&`iAoddlZddlZddlZddlmZmZddlmZddlm Z m Z m Z m Z m Z mZmZmZmZmZmZddlZddlZddlZddlmZmZddlmZddlmZddl m!Z!e r ddl"Z"ddlZdd l#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*ed dZ+eddZ,edZ-edZ.edZ/ee0ddfZ1edZ2edZ3ej4e5Z6e!GddeZ7e!Gdde8eZ9edde e8ej:ffZ;ee2ej:fZee e+ge,fe e+ge e,fe0d fZ?eeee/d!fZ@ed!ZAgd"ZBd#ZCd$ee1d%eDfd&ZEd'e d(d%ed(fd)ZFd*ee8d%e8fd+ZGe!d,ed!d%ed-fd.ZHe!Gd/d0ZIGd1d2ZJe!eGd3d-ZKd4eeKDZLe!eGd5d!eKZMe!d67eGd8d9eMZNe!Gd:d;ZOe!d<7Gd=d>ZPd?eQej:d%ej:fd@ZRdS)AN) dataclassfields)Enum) TYPE_CHECKINGAnyCallableDictIteratorListOptionalProtocolTupleTypeVarUnion)_check_pyarrow_version_truncated_repr) ObjectRef)log_once) DeveloperAPI) BlockBuilderPandasBlockSchema)SortKey) AggregateFnTT) contravariantU) covariantKeyTypeAggType) pyarrow.Tablepandas.DataFramerpyarrow.lib.Schema)pyarrow.ChunkedArray pyarrow.Array pandas.Series)r&z np.ndarrayr%r$ceZdZdZdZdS) BlockTypearrowpandasN)__name__ __module__ __qualname__ARROWPANDASb/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/data/block.pyr(r(Ds E FFFr1r(ceZdZdZdZdZdS) BatchFormatpyarrowr*numpyN)r+r,r-r.r/NUMPYr0r1r2r4r4Js E F EEEr1r4r!r"c:eZdZdedeeeeffdZdS)_CallableClassProtocol_CallableClassProtocol__argreturncdSNr0)selfr:s r2__call__z_CallableClassProtocol.__call__`s r1N)r+r,r-rrrr r?r0r1r2r9r9_sC a E!Xa[.$9      r1r9 BlockMetadata)r*r5r6Nr6schemar;cLddlm}|dupt||r|j n| S)Nrr)ray.data._internal.pandas_blockr isinstancenames)rArs r2_is_empty_schemarFwsGAAAAAA T>  f/ 0 0 FLZr1schemasSchemac6|D]}t|s|cSdS)zReturn the first non-empty schema from an iterator of schemas. Args: schemas: Iterator of schemas to check. Returns: The first non-empty schema, or None if all schemas are empty. N)rF)rGrAs r2_take_first_non_empty_schemarJs6'' MMM  4r1given_batch_formatch|dkrt}|tvrtd|dtd|S)NdefaultzThe given batch format z isn't allowed (must be one of z).)DEFAULT_BATCH_FORMATVALID_BATCH_FORMATS ValueError)rKs r2_apply_batch_formatrQs]Y&&1!444 (&8 ( (# ( ( (    r1metas BlockStatscd|DS)Nc6g|]}|Sr0)to_stats).0ms r2 zto_stats..s ( ( (QAJJLL ( ( (r1r0)rRs r2rVrVs ( (% ( ( ((r1c6eZdZdZdZeddZdZdS) BlockExecStatsaExecution stats for this block. Attributes: wall_time_s: The wall-clock time it took to compute this block. cpu_time_s: The CPU time it took to compute this block. node_id: A unique id for the node that computed this block. max_uss_bytes: An estimate of the maximum amount of physical memory that the process was using while computing this block. cd|_d|_d|_d|_d|_t j|_ d|_ d|_ dS)Nr) start_time_s end_time_s wall_time_s udf_time_s cpu_time_srayruntime_contextget_runtime_context get_node_idnode_id max_uss_bytestask_idxr>s r2__init__zBlockExecStats.__init__s\-1+/,0+,+/*>>@@LLNN "#'+ r1r;_BlockExecStatsBuilderctSr=)rkr0r1r2builderzBlockExecStats.builders%'''r1cRt|j|j|j|jdS)N)r_rar`rf)reprr_rar`rfris r2__repr__zBlockExecStats.__repr__s3#/"o"o<      r1N)r;rk)r+r,r-__doc__rj staticmethodrmrpr0r1r2r[r[s\,,,(((\(     r1r[c eZdZdZdZddZdS)rkzHelper class for building block stats. When this class is created, we record the start time. When build() is called, the time delta is saved as part of the stats. cftj|_tj|_dSr=)time perf_counter _start_time process_time _start_cpuris r2rjz_BlockExecStatsBuilder.__init__s',..+--r1r;r[ctj}tj}t}|j|_||_||jz |_||jz |_ |Sr=) rurvrxr[rwr]r^r_ryra)r>end_timeend_cpustatss r2buildz_BlockExecStatsBuilder.buildsb$&&#%%  !-#$t'77"T_4 r1N)r;r[)r+r,r-rqrjr~r0r1r2rkrksA ...      r1rkc\eZdZUdZeeed<eeed<eeed<dZdS)rSz#Statistics about the block producednum_rows size_bytes exec_statscP|jt|jtsJdSdSr=)rrDintris r2 __post_init__zBlockStats.__post_init__s4 ? &dos33 3 33 ' & 4 3r1N) r+r,r-rqr r__annotations__r[rr0r1r2rSrSs`.-sm ((((44444r1ch|] }|j Sr0)name)rWfs r2 rs???qAF???r1cLeZdZUdZeeeed<dZfdZ xZ S)r@zMetadata about the block. input_filesc>tdifdtDS)Nc<i|]}||Sr0)__getattribute__)rWkeyr>s r2 z*BlockMetadata.to_stats..s)SSS3sD))#..SSSr1r0)rS_BLOCK_STATS_FIELD_NAMESris`r2rVzBlockMetadata.to_statss7  SSSS:RSSS   r1cht|j g|_dSdSr=)superrr)r> __class__s r2rzBlockMetadata.__post_init__s8    #!D    $ #r1) r+r,r-rqr r strrrVr __classcell__rs@r2r@r@sh$# $s)$$$$   """""""""r1alpha) stabilityceZdZUdZeeed<d dededffd Z d de dedd dfd Z e d efd Z xZ S) BlockMetadataWithSchemaNrAmetadatarHct|j|j|j|j||_dS)N)rrrr)rrjrrrrrA)r>rrArs r2rjz BlockMetadataWithSchema.__init__sI  ,*&*     r1blockr}r[r;ct|}||}|}t ||S)N)r)rrA) BlockAccessor for_block get_metadatarAr)rr}accessormetarAs r2 from_blockz"BlockMetadataWithSchema.from_blocksO!**511$$$66""&VDDDDr1cPt|j|j|j|jS)N)rrrr)r@rrrrris r2rz BlockMetadataWithSchema.metadata!s.](     r1r=)r+r,r-rAr rHrr@rjBlockrpropertyrrrs@r2rr s $FHV ###8J;?EEE%&67E "EEEE -   X     r1rc2eZdZdZdefdZdedeefdZ dQdeded ede fd Z d e ede fd Z d e ede fdZd e eede fdZdeeefde fdZdedede fdZdeede fdZdRdZ dSd eeee efdeejeeejfffdZdTdZde fdZde fdZdeedefd Z defd!Z!dee"d"ffd#Z# dUd$ee ed%ee$de%fd&Z&dVd)Z'e(dWd+Z)e* dSd,ed-ee+de fd.Z,e*d,eee-fde fd/Z.e*d,eee-fde fd0Z/e(d1e dd2fd3Z0d4ed5d6dd(fd7Z1dQd8ed9edee2fd:Z3d8ed9edee2fd;Z4d8ed9edee2fd<Z5d8ed9edee2fd=Z6d8ed9edee2fd>Z7 dSd8ed9ed?ee2dee2fd@Z8dXdAZ9dBe ed5d6de d(fdCZ:dDd6dEe;dFde fdGZe( dYdHe e d5d6dEe;dFdKede;e e=ff dLZ?dBe e;e-d5d6de e fdMZ@de+fdNZAdOe edejfdPZBdS)ZraProvides accessor methods for a specific block. Ideally, we wouldn't need a separate accessor classes for blocks. However, this is needed if we want to support storing ``pyarrow.Table`` directly as a top-level Ray object, without a wrapping class (issue #17186). r;ct)z2Return the number of rows contained in this block.NotImplementedErrorris r2rzBlockAccessor.num_rows4!!r1public_row_formatct)zIterate over the rows of this block. Args: public_row_format: Whether to cast rows into the public Dict row format (this incurs extra copy conversions). r)r>rs r2 iter_rowszBlockAccessor.iter_rows8 "!r1Fstartendcopyct)a(Return a slice of this block. Args: start: The starting index of the slice (inclusive). end: The ending index of the slice (exclusive). copy: Whether to perform a data copy for the slice. Returns: The sliced block result. r)r>rrrs r2slicezBlockAccessor.sliceAs "!r1indicesct)zReturn a new block containing the provided row indices. Args: indices: The row indices to return. Returns: A new block containing the provided row indices. r)r>rs r2takezBlockAccessor.takeNs "!r1columnsct)zrs r2dropzBlockAccessor.dropYrr1ct)z3Return a new block containing the provided columns.rrs r2selectzBlockAccessor.select]rr1columns_renamect)z0Return the block reflecting the renamed columns.r)r>rs r2rename_columnszBlockAccessor.rename_columnsarr1 column_name column_datact)aq Upserts a column into the block. If the column already exists, it will be replaced. Args: column_name: The name of the column to upsert. column_data: The data to upsert into the column. (Arrow Array/ChunkedArray for Arrow blocks, Series or array-like for Pandas blocks) Returns: The updated block. r)r>rrs r2 upsert_columnzBlockAccessor.upsert_columnes"###r1 random_seedct)zRandomly shuffle this block.r)r>rs r2random_shufflezBlockAccessor.random_shufflerrr1r"ct)z+Convert this block into a Pandas dataframe.rris r2 to_pandaszBlockAccessor.to_pandasvrr1Nct)zConvert this block (or columns of block) into a NumPy ndarray. Args: columns: Name of columns to convert, or None if converting all columns. rrs r2to_numpyzBlockAccessor.to_numpyz "!r1r!ct)z'Convert this block into an Arrow table.rris r2to_arrowzBlockAccessor.to_arrowrr1ct)z/Return the base block that this accessor wraps.rris r2to_blockzBlockAccessor.to_blockrr1c*|S)z1Return the default data format for this accessor.)rris r2 to_defaultzBlockAccessor.to_defaults}}r1 batch_formatc>||S|dks|dkr|S|dkr|S|dkr|S|dkr|St dt d|) zConvert this block into the provided batch format. Args: batch_format: The batch format to convert this block to. Returns: This block formatted as the provided batch format. NrMnativer*r5r6z The batch format must be one of z, got: )rrrrrrPrO)r>rs r2to_batch_formatzBlockAccessor.to_batch_formats  ==?? " Y & &,(*B*B??$$ $ X % %>>## # Y & &==?? " W $ $==?? ""3F"""" r1ct)z3Return the approximate size in bytes of this block.rris r2rzBlockAccessor.size_bytesrr1r#ct)z7Return the Python type or pyarrow schema of this block.rris r2rAzBlockAccessor.schemarr1rrcpt||||S)z)Create a metadata object from this block.)rrrr)r@rr)r>rrs r2rzBlockAccessor.get_metadatas9 ]]__((#!     r1otherrct)zrs r2zipzBlockAccessor.ziprr1rct)z%Create a builder for this block type.rr0r1r2rmzBlockAccessor.builders "!r1batch block_typect|tjr tdt |dt|t jjr||tj krtddl m } | |S#|$rP}tdrtd|d|||cYd}~S|d}~wwxYw|tjksJ||S|S) z-Create a block from user-facing data formats.Error validating z: Standalone numpy arrays are not allowed in Ray 2.5. Return a dict of field -> array, e.g., `{'data': array}` instead of `array`.Nr)ArrowConversionError!_fallback_to_pandas_block_warningz)Failed to convert batch to Arrow due to: z; falling back to Pandas block)rDnpndarrayrPr collectionsabcMappingr(r.$ray.air.util.tensor_extensions.arrowrbatch_to_arrow_blockrloggerwarningbatch_to_pandas_blockr/)clsrrres r2batch_to_blockzBlockAccessor.batch_to_blocksY eRZ ( ( 8>OE$:$:>>> {6 7 7 8!Z9?%B%BUUUUUU  33E:::+     CDD<<<< ")"88????????  "Y%5555500777 s%3BC ACCCCc8ddlm}||S)z4Create an Arrow block from user-facing data formats.r)ArrowBlockBuilder)ray.data._internal.arrow_blockr_table_from_pydict)rrrs r2rz"BlockAccessor.batch_to_arrow_blocks+ EDDDDD 33E:::r1c8ddlm}||S)z4Create a Pandas block from user-facing data formats.r)PandasBlockBuilder)rCrr)rrrs r2rz#BlockAccessor.batch_to_pandas_blocks+ GFFFFF!44U;;;r1rzBlockAccessor[T]ctddl}ddl}t||j|jfrddlm}||St||jrddl m }||St|trddlm}| |St|tr tdt|dt!d|t%|)z,Create a block accessor for the given block.rN)ArrowBlockAccessor)PandasBlockAccessorrz: Standalone Python objects are not allowed in Ray 2.5. To use Python objects in a dataset, wrap them in a dict of numpy arrays, e.g., return `{'item': batch}` instead of just `batch`.zNot a block type: {} ({}))rr*r5rDTable RecordBatchrr DataFramerCrbytes from_byteslistrPr TypeErrorformattype)rr*r5rrs r2rzBlockAccessor.for_blocksB      egmW-@A B B T I I I I I I%%e,, , v/ 0 0 T K K K K K K&&u-- - u % % T I I I I I I%0077 7 t $ $ TDOE$:$:DDD 7>>ud5kkRRSS Sr1 n_samplessort_keyrct)z0Return a random sample of items from this block.r)r>rrs r2samplezBlockAccessor.samplerr1on ignore_nullsct)z=Returns a count of the distinct values in the provided columnrr>r r s r2countzBlockAccessor.countrr1ct)z2Returns a sum of the values in the provided columnrr s r2sumzBlockAccessor.sum!rr1ct)z2Returns a min of the values in the provided columnrr s r2minzBlockAccessor.min%rr1ct)z2Returns a max of the values in the provided columnrr s r2maxzBlockAccessor.max)rr1ct)z3Returns a mean of the values in the provided columnrr s r2meanzBlockAccessor.mean-rr1rct)zBReturns a sum of diffs (from mean) squared for the provided columnr)r>r r rs r2sum_of_squared_diffs_from_meanz,BlockAccessor.sum_of_squared_diffs_from_mean1rr1ct)z9Returns new block sorted according to provided `sort_key`r)r>rs r2sortzBlockAccessor.sort:rr1 boundariesct)z1Return a list of sorted partitions of this block.rr>rrs r2sort_and_partitionz BlockAccessor.sort_and_partition>s "!r1raggsrct)z3Combine rows with the same key into an accumulator.r)r>rrs r2 _aggregatezBlockAccessor._aggregateDrr1blocksct)z9Return a sorted block by merging a list of sorted blocks.r)r"rs r2merge_sorted_blocksz!BlockAccessor.merge_sorted_blocksHs "!r1Tfinalizect)z/Aggregate partially combined and sorted blocks.r)r"rrr%s r2_combine_aggregated_blocksz(BlockAccessor._combine_aggregated_blocksOrr1ct)zNOTE: PLEASE READ CAREFULLY Returns dataset partitioned using list of boundaries This method requires that - Block being sorted (according to `sort_key`) - Boundaries is a sorted list of tuples rrs r2_find_partitions_sortedz%BlockAccessor._find_partitions_sortedYs "!r1ct)z$Return the block type of this block.rris r2rzBlockAccessor.block_typehrr1keyscL|dkr tjgtjS|s(tjd|gS||}t t |S)a NOTE: THIS METHOD ASSUMES THAT PROVIDED BLOCK IS ALREADY SORTED Compute boundaries of the groups within a block based on provided key (a column or a list of columns) NOTE: In each column, NaNs/None are considered to be the same group. Args: block: sorted block for which grouping of rows will be determined based on provided key keys: list of columns determining the key for every row based on which the block will be grouped Returns: A list of starting indices of each group and an end index of the last group, i.e., there are ``num_groups + 1`` entries and the first and last entries are 0 and ``len(array)`` respectively. r)dtype)rrarrayint32r"_get_group_boundaries_sorted_numpyrvalues)r>r+projected_blocks r2_get_group_boundaries_sortedz*BlockAccessor._get_group_boundaries_sortedls* ==??a  8Bbh/// / 28Q 011 1----1$7M7M7O7O2P2PQQQr1F)r;r"r=)r;r!)NN)rrr;r)r;r)rrr;r)T)Cr+r,r-rqrrboolr rrrrr rrrr rr r BlockColumnrrrrrrrrrr DataBatchrrrrAr[r@rrrrrm classmethodr(rrrrrr rrrrrrrrrrr!rr$r'r)rr3r0r1r2rr+s"#"""""4"HQK"""" " "3 "S " " " " " " "DI "% " " " ""DI"%"""""d8C=1"e"""""T#s(^""""" $ $; $5 $ $ $ $"(3-"E"""""""" :>""c49n 56" rz4RZ00 1"""""""""%""""EHSMi2"C"""""d$889"""" ,0/3   d3i(  ^,        """""""\"+/###Y'#  ###[#J;c3h;E;;;[; <$sCx.r<s r2rjzBlockColumnAccessor.__init__s  r1T)as_pyr r?r;ct)z4Returns a count of the distinct values in the columnrr>r r?s r2rzBlockColumnAccessor.count!###r1ctS)z)Returns a sum of the values in the columnrrAs r2rzBlockColumnAccessor.sums"$$$r1ct)z)Returns a min of the values in the columnrrAs r2rzBlockColumnAccessor.minrBr1ct)z)Returns a max of the values in the columnrrAs r2rzBlockColumnAccessor.maxrBr1ct)z*Returns a mean of the values in the columnrrAs r2rzBlockColumnAccessor.meanrBr1qct)z.Returns requested quantile of the given columnr)r>rGr r?s r2quantilezBlockColumnAccessor.quantiles"###r1ct)zBReturns new column holding only distinct values of the current onerris r2uniquezBlockColumnAccessor.uniquerBr1ctr=rris r2 value_countsz BlockColumnAccessor.value_counts!###r1ct)a Computes a 64-bit hash value for each row in the column. Provides a unified hashing method across supported backends. Handles complex types like lists or nested structures by producing a single hash per row. These hashes are useful for downstream operations such as deduplication, grouping, or partitioning. Internally, Polars is used to compute row-level hashes even when the original column is backed by Pandas or PyArrow. Returns: A column of 64-bit integer hashes, returned in the same format as the underlying backend (e.g., Pandas Series or PyArrow Array). rris r2hashzBlockColumnAccessor.hashs"###r1ct)z;Flattens nested lists merging them into top-level containerrris r2flattenzBlockColumnAccessor.flattens"###r1ctr=rris r2dropnazBlockColumnAccessor.dropnarNr1ct)z Checks whether the column is composed of list-like elements. Returns: True if the column is made up of list-like values; False otherwise. rris r2is_composed_of_listsz(BlockColumnAccessor.is_composed_of_listss"###r1N)rr?rct)z9Returns a sum of diffs (from mean) squared for the columnr)r>r rr?s r2rz2BlockColumnAccessor.sum_of_squared_diffs_from_means"###r1ct)z8Converts block column to a list of Python native objectsrris r2 to_pylistzBlockColumnAccessor.to_pylistrBr1Fzero_copy_onlyct)z#Converts underlying column to Numpyr)r>rZs r2rzBlockColumnAccessor.to_numpyrBr1r%ct)zAConverts block column into a representation compatible with Arrowrris r2_as_arrow_compatiblez(BlockColumnAccessor._as_arrow_compatiblerBr1c<tddl}t|tjst|tjrddlm}||St||jrddl m }||Stdt|d)z-Create a column accessor for the given columnrN)ArrowBlockColumnAccessor)PandasBlockColumnAccessorzEExpected either a pandas.Series or pyarrow.Array (ChunkedArray) (got )) rr*rDpaArray ChunkedArrayrr_SeriesrCr`rr)r<pdr_r`s r2 for_columnzBlockColumnAccessor.for_columns     c28 $ $  3(H(H  O O O O O O++C00 0 RY ' '  Q Q Q Q Q Q,,S11 1%S %%% r1r4)#r+r,r-rqr6rjr5r rrrrrrfloatrIrKr rr rMrPrRrTrVrrrYrrrrr]rrrgr0r1r2r;r;sM))K:>$$$T$$$(1+$$$$8<%%%4%% %%%%8<$$$4$$ $$$$8<$$$4$$ $$$$9=$$$D$$!$$$$ >B$$$$)-$6:$ !$$$$ $ $$$$$d39o$$$$$k$$$$"$$$$$ $ $$$$$d$$$$! $$$$qk $  $ ! $$$$$49$$$$$$t$ $$$$$eDI,F&G$$$$ (=\r1r;rcg}g}g}|D]}tj|jtjr0tj|dr||Vtj|jtjs|d||||g}t |dkrE|tjd|Ddt |dkrE|tjd|Ddt |dkrE|tjd|Ddtj dgtj |d ddzt |dgg t}|S)Nrc:g|]}|dd|ddkSNrjr0rWarrs r2rYz6_get_group_boundaries_sorted_numpy..s-EEEss122w#crc(*EEEr1)axisc g|]R}|dd|ddktj|ddtj|ddzzSSrl)risfiniterns r2rYz6_get_group_boundaries_sorted_numpy.."sqWCRC({3qrr7++bk#crc(.C.CCEr1c g|]U}|dd|ddktj|dddtj|dddzzVSrl)requalrns r2rYz6_get_group_boundaries_sorted_numpy../sxWCRC(QRR$//"(3ss8T2J2JJKLr1rm)r issubdtyper-numberisnanappendlenvstackanyhstack column_stacknonzeroastyper)rgeneral_arraysnum_arrays_with_nancat_arrays_with_nonerodiffsrs r2r0r0saN'' =BI . . '28CG3D3D '  & &s + + + +sy")44 'R ' ' , , , ,  ! !# & & & & E >Q IEEnEEE F F J JPQ J R R    !##  I 3   cqckk      1$$  I 4   cqckk    C _U # # ' 'Q ' / / 7 7 9 9! rsK ))))))))                           KKKKKKKK------/MMMNNN======AAAAAAJJJJJJ...... GCt$$$ GC4    ')   ')   12 t(*>> ?LM J  8 $ $ #t /#5tCO7LL M  RZ/0      Xad^    aS!V aS(1+  !"$eIe,o=>?o.:::Xf-$ (8*< (AS    HSMc)D))d<.@))))! ! ! ! ! ! ! ! H4 4444444 4"@?FF:,>,>??? """""J"" "(         m   !  <_R_R_R_R_R_R_R_RD sssssss sl?RZ0@?RZ??????r1