
    %`iKl                        d dl Z d dlZd dlZd dlmZ d dlmZmZmZm	Z	 d dl
mZ erd dlZdZdZ e j        e          Z e            ad Zd Zd	 Zd
 ZdGdZdee	ed         df                  ddddfdZddde	ed         df         fdZded         ddddfdZe G d d                      ZdHdZdId Zdddefd!Z dddefd"Z!dJd$Z"dId%Z#dId&Z$dId'Z%	 	 	 	 dKd)Z&dLd+Z'dMd-Z(	 	 	 	 dNd/Z)dOd1Z*	 	 	 	 dPd3Z+d4d5ded         d6e,d7e,dd5f
d8Z-d4d5d9e,d6e,d7e,dd5f
d:Z.d4d5d6e,d7e,dd5fd;Z/d4d5d<dd6e,d7e,de	d5e,e,f         f
d=Z0d>e,de,fd?Z1d4d5d@e,dAe,dd5fdBZ2dQdDZ3d4e4ddfdEZ5dddefdFZ6dS )R    N)	dataclass)TYPE_CHECKINGListOptionalTuple)
is_in_test3RAY_DISABLE_CUSTOM_ARROW_JSON_OPTIONS_SERIALIZATION+RAY_DISABLE_CUSTOM_ARROW_DATA_SERIALIZATIONc                     	 dd l }n# t          $ r Y d S w xY wt          |            t          |            t	          |            d S )Nr   )pyarrowModuleNotFoundError_register_arrow_data_serializer+_register_arrow_json_readoptions_serializer,_register_arrow_json_parseoptions_serializerserialization_contextpas     t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_private/arrow_serialization.py%_register_custom_datasets_serializersr      sm       
 $$9:::/0EFFF01FGGGGGs    
c                     t           j                            t          d          dk    rd S dd lm |                     j        d fd           d S )N01r   c                     | j         | j        fS N)use_threads
block_sizeoptss    r   <lambda>z=_register_arrow_json_readoptions_serializer.<locals>.<lambda>:   s    (8$/'J     c                      j         |  S r   )ReadOptionsargspajsons    r   r   z=_register_arrow_json_readoptions_serializer.<locals>.<lambda>;   s    ););T)B r    custom_serializercustom_deserializer)osenvirongetr	   pyarrow.jsonjson _register_cloudpickle_serializerr"   r   r%   s    @r   r   r   ,   s    

?	
 	
 		 	 	!!!!!!::JJBBBB ;     r    c                     t           j                            t          d          dk    rd S dd lm |                     j        d fd           d S )Nr   r   r   c                 *    | j         | j        | j        fS r   )explicit_schemanewlines_in_valuesunexpected_field_behaviorr   s    r   r   z>_register_arrow_json_parseoptions_serializer.<locals>.<lambda>M   s     #*(
 r    c                      j         |  S r   )ParseOptionsr#   s    r   r   z>_register_arrow_json_parseoptions_serializer.<locals>.<lambda>R   s    )<)<d)C r    r&   )r)   r*   r+   r	   r,   r-   r.   r6   r/   s    @r   r   r   ?   s    

?	
 	
 		 	 	!!!!!!::
 

 DCCC ;     r    c                     t           j                            t          d          dk    rdS ddl}|                     |j        t                     dS )ay  Custom reducer for Arrow data that works around a zero-copy slicing pickling
    bug by using the Arrow IPC format for the underlying serialization.

    Background:
        Arrow has both array-level slicing and buffer-level slicing; both are zero-copy,
        but the former has a serialization bug where the entire buffer is serialized
        instead of just the slice, while the latter's serialization works as expected
        and only serializes the slice of the buffer. I.e., array-level slicing doesn't
        propagate the slice down to the buffer when serializing the array.

        We work around this by registering a custom cloudpickle reducers for Arrow
        Tables that delegates serialization to the Arrow IPC format; thankfully, Arrow's
        IPC serialization has fixed this buffer truncation bug.

    See https://issues.apache.org/jira/browse/ARROW-10739.
    r   r   Nr   )r)   r*   r+   r
   r   _register_cloudpickle_reducerTable_arrow_table_reducer   s     r   r   r   X   sP    " 
z~~A3GG3NN77BUVVVVVr    tpyarrow.Tablec           	         g }| j         D ]}| |         }	 t          |          }|                    |           0# t          $ r}t	          |j                  st                      r|dt          |j                  t          vrTt          	                    d| d|j         dd           t          
                    t          |j                             t          |           cY d}~c S d}~ww xY wt          || j        ffS )a/  Custom reducer for Arrow Tables that works around a zero-copy slice pickling bug.
    Background:
        Arrow has both array-level slicing and buffer-level slicing; both are zero-copy,
        but the former has a serialization bug where the entire buffer is serialized
        instead of just the slice, while the latter's serialization works as expected
        and only serializes the slice of the buffer. I.e., array-level slicing doesn't
        propagate the slice down to the buffer when serializing the array.
        All that these copy methods do is, at serialization time, take the array-level
        slicing and translate them to buffer-level slicing, so only the buffer slice is
        sent over the wire instead of the entire buffer.
    See https://issues.apache.org/jira/browse/ARROW-10739.
    NzTFailed to complete optimized serialization of Arrow Table, serialization of column 'z
' of type z failed, so we're falling back to Arrow IPC serialization for the table. Note that this may result in slower serialization and more worker memory utilization. Serialization error:T)exc_info)column_names_arrow_chunked_array_reduceappend	Exception_is_dense_uniontyper   _serialization_fallback_setloggerwarningadd_arrow_table_ipc_reduce_reconstruct_tableschema)r;   reduced_columnscolumn_namecolumnreduced_columnes         r   r:   r:   q   sO     O~ 3 3;	38@@N* "">2222)  	. 	. 	."6;// "JLL " T!FK  (CCCF0;F FGM{F F F
 "     ,//V[0A0ABBB*1----------#	.* :::s   :
C3B"C.&C3.C3rL   pyarrow.Arraypyarrow.DataTyperK   zpyarrow.Schemareturnc                     ddl }g }| D ](\  }}|                    t          ||                     )|j                            ||          S )zERestore a serialized Arrow Table, reconstructing each reduced column.r   NrK   )r   rA   _reconstruct_chunked_arrayr9   from_arrays)rL   rK   r   columnschunks_payloadtype_s         r   rJ   rJ      sh    
  G!0 J J1.%HHIIII8777r    cazpyarrow.ChunkedArrayPicklableArrayPayloadc                     g }| j         D ]1}t                              |          }|                    |           2|| j        fS )zCustom reducer for Arrow ChunkedArrays that works around a zero-copy slice
    pickling bug. This reducer does not return a reconstruction function, since it's
    expected to be reconstructed by the Arrow Table reconstructor.
    )chunksr\   
from_arrayrA   rD   )r[   chunk_payloadschunkchunk_payloads       r   r@   r@      sQ     N - --88??m,,,,27""r    r^   rZ   c                 N    ddl }d | D             } |                    | |          S )z=Restore a serialized Arrow ChunkedArray from chunks and type.r   Nc                 6    g | ]}|                                 S  to_array).0ra   s     r   
<listcomp>z._reconstruct_chunked_array.<locals>.<listcomp>   s"    3335enn333r    )r   chunked_array)r^   rZ   r   s      r   rV   rV      s;      43F333FFE***r    c                       e Zd ZU dZded<   eed<   ed         ed<   eed<   eed<   ed          ed	<   edd            ZddZ	dS )r\   zPicklable array payload, holding data buffers and array metadata.

    This is a helper container for pickling and reconstructing nested Arrow Arrays while
    ensuring that the buffers that underly zero-copy slice views are properly truncated.
    rR   rD   lengthpyarrow.Bufferbuffers
null_countoffsetchildrenarQ   rS   c                      t          |          S )a  Create a picklable array payload from an Arrow Array.

        This will recursively accumulate data buffer and metadata payloads that are
        ready for pickling; namely, the data buffers underlying zero-copy slice views
        will be properly truncated.
        )_array_to_array_payload)selfrr   s     r   r_   z PicklableArrayPayload.from_array   s     'q)))r    c                      t          |           S )z7Reconstruct an Arrow Array from this picklable payload.)_array_payload_to_array)ru   s    r   rg   zPicklableArrayPayload.to_array   s    &t,,,r    Nrr   rQ   rS   r\   )rS   rQ   )
__name__
__module____qualname____doc____annotations__intr   classmethodr_   rg   re   r    r   r\   r\      s           KKK"####OOOKKK*++++* * * [*- - - - - -r    payloadc                 8   ddl }d | j        D             }|j                            | j                  rJt          |          dk    sJ t          |                      |\  }}|j                            ||          S |j                            | j                  r_t          |          dk    rLt          |          dk    sJ t          |                      |\  }}}|j	                            |||          S t          | j        |j                  rLt          |          dk    sJ t          |                      |d         }| j                            |          S |j                            | j        | j        | j        | j        | j        |          S )zHReconstruct an Arrow Array from a possibly nested PicklableArrayPayload.r   Nc                 6    g | ]}|                                 S re   rf   )rh   child_payloads     r   ri   z+_array_payload_to_array.<locals>.<listcomp>   s$    OOO]&&((OOOr             rD   rl   rn   ro   rp   rq   )r   rq   typesis_dictionaryrD   lenDictionaryArrayrW   is_mapMapArray
isinstanceBaseExtensionType
wrap_arrayArrayfrom_buffersrl   rn   ro   rp   )	r   r   rq   indices
dictionaryoffsetskeysitemsstorages	            r   rw   rw      s   OOg>NOOOH	xgl++ 
 8}}!!!3x==!!!&!--gzBBB		&	& 
3x==1+<+< 8}}!!!3x==!!!'u{&&we<<<	GL""6	7	7 
8}}!!!3x==!!!1+|&&w/// x$$>O)> % 
 
 	
r    rr   c                    ddl }t          | j                  rt          d          |j                            | j                  rt          |           S t          | j                  rt          |           S t          | j                  rt          |           S |j                            | j                  s|j                            | j                  rt          |           S |j                            | j                  rt          |           S |j                            | j                  rt#          |           S |j                            | j                  rt'          |           S |j                            | j                  rt+          |           S |j                            | j                  rt/          |           S t1          | j        |j                  rt5          |           S t7          d| j                  )zSerialize an Arrow Array to an PicklableArrayPayload for later pickling.

    This function's primary purpose is to dispatch to the handler for the input array
    type.
    r   NzKCustom slice view serialization of dense union arrays is not yet supported.zUnhandled Arrow array type:)r   rC   rD   NotImplementedErrorr   is_null_null_array_to_array_payload_is_primitive!_primitive_array_to_array_payload
_is_binary_binary_array_to_array_payloadis_listis_large_list_list_array_to_array_payloadis_fixed_size_list'_fixed_size_list_array_to_array_payload	is_struct_struct_array_to_array_payloadis_union_union_array_to_array_payloadr   "_dictionary_array_to_array_payloadr   _map_array_to_array_payloadr   r   !_extension_array_to_array_payload
ValueError)rr   r   s     r   rt   rt     s    qv 
 "
 
 	
 
x @+A...	qv		 @0333	AF		 @-a000			!&	!	! @RX%;%;AF%C%C @+A...		$	$QV	,	, @6q999			AF	#	# @-a000			16	"	" 	@,Q///				'	' @1!444		 	  @*1---	AFB0	1	1 @03336???r    c                 B   ddl }|j                            |           p|j                            |           pg|j                            |           pM|j                            |           p3|j                            |           p|j                            |           S )zcWhether the provided Array type is primitive (boolean, numeric, temporal or
    fixed-size binary).r   N)r   r   
is_integeris_floating
is_decimal
is_booleanis_temporalis_fixed_size_binaryrZ   r   s     r   r   r   8  s      	E"" 	08&&	08u%%	0 8u%%	0 8&&		0
 8((//r    c                     ddl }|j                            |           pM|j                            |           p3|j                            |           p|j                            |           S )z@Whether the provided Array type is a variable-sized binary type.r   N)r   r   	is_stringis_large_string	is_binaryis_large_binaryr   s     r   r   r   G  sr     	5!! 	+8##E**	+8e$$	+ 8##E**	r    pyarrow.NullArrayc                 \    t          | j        t          |           dg| j        dg           S )z.Serialize null array to PicklableArrayPayload.Nr   r   )r\   rD   r   ro   )rr   s    r   r   r   S  s9     !V1vv<   r    c                    t          | j                  sJ | j                    |                                 }t          |          dk    sJ t          |                      |d         }| j        dk    r$t          || j        t          |                     }nd}|d         }|/t          |d         | j        | j        t          |                     }t          | j        t          |           ||g| j        dg           S )zZSerialize primitive (numeric, temporal, boolean) arrays to
    PicklableArrayPayload.
    r   r   Nr   r   )	r   rD   rn   r   ro    _copy_bitpacked_buffer_if_neededrp   _copy_buffer_if_neededr\   )rr   rn   
bitmap_bufdata_bufs       r   r   r   `  s       ((!&(( iikkGw<<1c'll J|a5j!(CPQFFSS


 qzH)'!*afahAOO V1vvX&<   r    c                 .   t          | j                  sJ | j                    |                                 }t          |          dk    sJ t          |                      | j        dk    r*t          |d         | j        t          |                     }nd}|d         }t          || j        | j        t          |                     \  }}}|d         }t          |d||          }t          | j        t          |           |||g| j        dg           S )zZSerialize binary (variable-sized binary, string) arrays to
    PicklableArrayPayload.
    r   r   Nr   r   r   )
r   rD   rn   r   ro   r   rp   _copy_offsets_buffer_if_neededr   r\   )rr   rn   r   
offset_bufdata_offsetdata_lengthr   s          r   r   r     s    af%%qv%%iikkGw<<1c'll 	|a5gaj!(CPQFFSS


 J+IAFAHc!ff, ,(J[ qzH%hk;OOH V1vvZ2<   r    c           
         |                                  }t          |          dk    sJ t          |                      | j        dk    r*t          |d         | j        t          |                     }nd}|d         }t          || j        | j        t          |                     \  }}}| j                            ||          }t          | j        t          |           ||g| j        dt          |          g          S )zCSerialize list (regular and large) arrays to PicklableArrayPayload.r   r   Nr   )rn   r   ro   r   rp   r   rD   valuesslicer\   rt   )rr   rn   r   r   child_offsetchild_lengthchilds          r   r   r     s     iikkGw<<!S\\ 	|a5gaj!(CPQFFSS


 J-KAFAHc!ff. .*Jl
 HNN<66E V1vvZ(<)%001   r    pyarrow.FixedSizeListArrayc           
         |                                  }t          |          dk    sJ t          |                      | j        dk    r*t          |d         | j        t          |                     }nd}| j        j        | j        z  }| j        j        t          |           z  }| j                            ||          }t          | j        t          |           |g| j        dt          |          g          S )z:Serialize fixed size list arrays to PicklableArrayPayload.r   r   Nr   )rn   r   ro   r   rp   rD   	list_sizer   r   r\   rt   )rr   rn   r   r   r   r   s         r   r   r     s     iikkGw<<1c'll 	|a5gaj!(CPQFFSS


 6#ah.L6#c!ff,LHNN<66E V1vv<)%001   r    pyarrow.StructArrayc                                                       }t          |          dk    sJ t          |                       j        dk    r*t          |d          j        t                               }nd} fdt           j        j                  D             }t           j        t                     |g j        d|          S )z1Serialize struct arrays to PicklableArrayPayload.r   r   Nc                 T    g | ]$}t                              |                    %S re   rt   fieldrh   irr   s     r   ri   z2_struct_array_to_array_payload.<locals>.<listcomp>  ,    VVV'

33VVVr    r   )	rn   r   ro   r   rp   rangerD   
num_fieldsr\   )rr   rn   r   rq   s   `   r   r   r     s    
 iikkGw<<1c'll 	|a5gaj!(CPQFFSS



 WVVVU16CT=U=UVVVH V1vv<   r    pyarrow.UnionArrayc                     ddl }t           j                  rJ                                  }t	          |          dk    sJ t	          |                      |d         }|
J |            |d         }t          ||                                 j        t	                               } fdt           j        j	                  D             }t           j        t	                     ||g j        d|          S )z0Serialize union arrays to PicklableArrayPayload.r   Nr   c                 T    g | ]$}t                              |                    %S re   r   r   s     r   ri   z1_union_array_to_array_payload.<locals>.<listcomp>  r   r    r   )r   rC   rD   rn   r   r   int8rp   r   r   r\   ro   )rr   r   rn   r   type_code_bufrq   s   `     r   r   r     s    qv&&&&&iikkGw<<!S\\Jz AJM*="''))QXsSTvvVVM
 WVVVU16CT=U=UVVVH V1vv]+<   r    pyarrow.DictionaryArrayc           	          t          | j                  }t          | j                  }t          | j        t          |           g | j        d||g          S )z5Serialize dictionary arrays to PicklableArrayPayload.r   r   )rt   r   r   r\   rD   r   ro   )rr   indices_payloaddictionary_payloads      r   r   r     s\     .ai88O0>> V1vv<!#56   r    pyarrow.MapArrayc                 .   ddl }|                                 }t          |          dk    sJ t          |                      | j        dk    r*t	          |d         | j        t          |                     }nd}|g}|d         }t          || j        | j        t          |                     \  }}}t          | |j	        j
                  r?|                    |           t          | j                            ||                    g}n|                                 }t          |          dk    sJ t          |                      |j                            |                                t          |           dz   ||g          }	| j                            ||          }
| j                            ||          }t          |	          t          |
          t          |          g}t)          | j        t          |           || j        d|          S )z.Serialize map arrays to PicklableArrayPayload.r   Nr   r   r   )r   rn   r   ro   r   rp   r   rD   r   lib	ListArrayrA   rt   r   r   r   r   int32r   r   r\   )rr   r   rn   r   new_buffersr   r   r   rq   r   r   r   s               r   r   r   /  s    iikkGw<<!S\\ 	|a5gaj!(CPQFFSS


,K J+IAFAHc!ff, ,(J[ !RV%&& 
 	:&&&+AHNN;,T,TUUV ))++7||aW(''HHJJA
Z$<
 
 v||K55k;77#G,,#D))#E**

 !V1vv<   r    pyarrow.ExtensionArrayc                     t          | j                  }t          | j        t	          |           g | j        d|g          S )Nr   r   )rt   r   r\   rD   r   ro   )rr   storage_payloads     r   r   r   g  sI     .ai88O V1vv<!"   r    bufrm   rp   rl   c                     ddl }|,|j                            |          rt          | ||          } n |
|j        dz  nd}t          | |||          } | S )Copy buffer, if needed.r   N   r   )r   r   r   r   	bit_width_copy_normal_buffer_if_needed)r   rZ   rp   rl   r   type_bytewidths         r   r   r   u  sn     RX0077 /sFFCC161BA--+CPPJr    
byte_widthc                 h    ||z  }||z  }|dk    s|| j         k     r|                     ||          } | S )r   r   )sizer   )r   r   rp   rl   byte_offsetbyte_lengths         r   r   r     sG     :%K:%Kzz[38++ ii[11Jr    c                     |dz  }|dz  }t          ||z             dz  }|dk    s|| j        k     r-|                     ||          } |dk    rt          | ||          } | S )z)Copy bit-packed binary buffer, if needed.r   r   )_bytes_for_bitsr   r   _align_bit_offset)r   rp   rl   
bit_offsetr   r   s         r   r   r     st     !JA+K!*v"566!;Kzz[38++ii[11??#C[AACJr    arr_typec                    ddl }ddlm} |j                            |          sN|j                            |          s4|j                            |          s|j                            |          r|                                }n|	                                }t          | |||dz             } |j                            ||dz   d| g          }|d                                         }|d                                         |z
  }	|                    ||          }|j                            |          r|                    |d          }|                                d         } | ||	fS )zvCopy the provided offsets buffer, returning the copied buffer and the
    offset + length of the underlying data.
    r   Nr   F)safe)r   pyarrow.computecomputer   r   r   r   is_large_unicodeint64r   r   r   r   as_pysubtractis_int32castrn   )
r   r   rp   rl   r   pacoffset_typer   r   r   s
             r   r   r     sp    !!!!!! 	x((!8##H--! 8##H--! 8$$X..	! hhjjhhjj
 k66A:
F
FC h##K!dC[IIG1:##%%L2;$$&&5Lll7L11G	x%% 8,,{,77
//

A
Cl**r    nc                     | dz   dz  S )zpRound up n to the nearest multiple of 8.
    This is used to get the byte-padded number of bits for n bits.
       ire   )r
  s    r   r   r     s     Eb>r    r   r   c                     ddl }|                                 }t                              |t          j                  }||z  }|                    |t          j                  }|                    |          S )z}Align the bit offset into the buffer with the front of the buffer by shifting
    the buffer and eliminating the offset.
    r   N)r   
to_pybytesr~   
from_bytessys	byteorderto_bytes	py_buffer)r   r   r   r   bytes_bytes_as_ints         r   r   r     sg     ^^F>>&#-88LZL"";>>F<<r    tablec                     ddl m} ddlm}  |            } ||| j                  5 }|                    |            ddd           n# 1 swxY w Y   t          |                                ffS )a4  Custom reducer for Arrow Table that works around a zero-copy slicing pickling
    bug by using the Arrow IPC format for the underlying serialization.

    This is currently used as a fallback for unsupported types (or unknown bugs) for
    the manual buffer truncation workaround, e.g. for dense unions.
    r   )RecordBatchStreamWriter)BufferOutputStreamrU   N)pyarrow.ipcr  pyarrow.libr  rK   write_table_restore_table_from_ipcgetvalue)r  r  r  output_streamwrs        r   rI   rI     s     433333......&&((M	 	 u|	D	D	D 
u              
 #]%;%;%=%=$???s   AAAc                     ddl m}  ||           5 }|                                cddd           S # 1 swxY w Y   dS )z6Restore an Arrow Table serialized to Arrow IPC format.r   )RecordBatchStreamReaderN)r  r"  read_all)r   r"  readers      r   r  r    s    333333	 	 	%	% !  ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   377c                 T    ddl }|j                            |           o
| j        dk    S )z1Whether the provided Arrow type is a dense union.r   Ndense)r   r   r   moder   s     r   rC   rC     s0    8U##=
g(==r    )r;   r<   )r   r\   rS   rQ   rx   )rr   r   rS   r\   )rr   r   rS   r\   )rr   r   rS   r\   )rr   r   rS   r\   )rr   r   rS   r\   )rr   r   rS   r\   )rr   r   rS   r\   )r  r<   )7loggingr)   r  dataclassesr   typingr   r   r   r   ray._private.utilsr   r   r	   r
   	getLoggerry   rF   setrE   r   r   r   r   r:   rJ   r@   rV   r\   rw   rt   boolr   r   r   r   r   r   r   r   r   r   r   r   r~   r   r   r   r   r   r   rI   bytesr  rC   re   r    r   <module>r0     s*  
  				 



 ! ! ! ! ! ! 7 7 7 7 7 7 7 7 7 7 7 7 ) ) ) ) ) ) NNN : 4 2 , 
	8	$	$ "cee 
H 
H 
H  &  2W W W2+; +; +; +;\8%_ 57I IJK88 8 8 8 8##
4'(*<<=# # # #	+()	+2D	+	+ 	+ 	+ 	+  -  -  -  -  -  -  -  -F
 
 
 
D&@ &@ &@ &@R+     	( 	T 	 	 	 	
 
 
 
   >   @   D#   >   8   B    $5 5 5 5p   	&'  	
    &	  	
    "	  	   ""+	"+ "+ "+ 	"+
 S#%&"+ "+ "+ "+Js s     	     	       "@ @ @ @&! !? ! ! ! !>- >$ > > > > > >r    