&`it JddlmZmZddlmZddlmZmZddlm Z m Z m Z m Z ddl mZdeededeefd Zd e d eed edee ee ffd Zdee deeed edeee eee ffdZdedeedee fdZdS))ListTuple) RefBundle)_calculate_blocks_rows_split_at_indices)Block BlockMetadataBlockPartition_take_first_non_empty_schema) ObjectRefper_split_bundlesowned_by_consumerreturnct|dkr|Sd|D}d|D}td|D}|t|z}t|||\}}}t||D]1\} } td| D} | |ksJ| | z|ksJ2t d|D} t ||| } t | |}t|D]G\}}|||td||D} | |ksJHg}|D]'}| t ||| (|S) zEqualize split ref bundles into equal number of rows. Args: per_split_bundles: ref bundles to equalize. Returns: the equalized ref bundles. rcg|] }|j S)blocks.0bundles o/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/data/_internal/equalize.py z_equalize..s%T%T%Tfm%T%T%Tc,g|]}t|Sr)r)rsplits rrz_equalize..s.+++*/u%%+++rc,g|]}t|Sr)sum)r blocks_rowss rrz_equalize..s MMM;c+&&MMMrc"g|] \}}|j Srnum_rowsr_metas rrz_equalize..*sIIIDt}IIIrc3$K|] }|jV dS)N)schemars r z_equalize..0s$)X)XF&-)X)X)X)X)X)Xr) owns_blocksr&c"g|] \}}|j Srr r"s rrz_equalize..9sMMMDt}MMMr) lenr_shave_all_splitszipr r_split_leftovers enumerateextendappend)r rper_split_blocks_with_metadataper_split_num_rows total_rowstarget_split_size shaved_splitsper_split_needed_rows leftovers shaved_splitsplit_needed_rownum_shaved_rowsr&leftover_bundleleftover_splitsileftover_splitequalized_ref_bundlesrs r _equalizer@s ""  %T%TBS%T%T%T"++3Q+++MM:LMMMNNJ"c*H&I&II7H&(:a///MMM!\}}||z|kr||||z })||?||z }|||fS)aShave a block list to the target size. Args: split: the block list to shave. num_rows_per_block: num rows for each block in the list. target_size: the upper bound target size of the shaved list. Returns: A tuple of: - shaved block list. - num of rows needed for the block list to meet the target size. - leftover blocks. r)r,r0) rrArBshavedr7 shaved_rowsblock_with_meta block_rowsnum_rows_neededs r_shave_one_splitrIEs"FIK'*52D'E'E..#  #{ 2 2 MM/ * * * : %KK   _ - - - -!K/O ?I --r input_splitsr2cg}g}g}t||D]Y\}}t|||\}} } |||| || Z|||fS)aShave all block list to the target size. Args: input_splits: all block list to shave. input_splits: num rows (per block) for each block list. target_size: the upper bound target size of the shaved lists. Returns: A tuple of: - all shaved block list. - num of rows needed for the block list to meet the target size. - leftover blocks. )r,rIr0r/) rJr2rBr5r6r7rrArDrH _leftoverss rr+r+cs"MI%(7I%J%J%%!!.> %{/ / + V$$$$$_555$$$$ / ::rr7r6ct|}g}d}t|D]%\}}|||z||}&t|j||j}dt |Dd|S)z0Split leftover blocks by the num of rows needed.rcNg|]"\}}tt||#Sr)listr,)r block_refsr$s rrz$_split_leftovers..s/ S S S,>  ?TO3444Q    T S\@R S S S  rN)typingrr'ray.data._internal.execution.interfacesrray.data._internal.splitrrray.data.blockrr r r ray.typesr boolr@intrIr+r-rrrr\s======NNNNNNNN  4!I4!4! )_4!4!4!4!n. ./3Cy.GJ. >3 ./....<;~&;T#Y;; 4 cN :; ;;;;@15c .r