
    &`i                     J
   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dl mZ d dlmZ d dl	m
Z
 d dlmZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmZmZ d dl m!Z! d d	l"m#Z#m$Z$m%Z% d d
l&m'Z'm(Z(m)Z)m*Z* d dl+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2 d dl3m4Z4  e              e            Z5 ed          Z6 ed          Z7dZ8 G d de
          Z9 e9 e!dd           dk    re9j:        ne9j;                  Z< e!dd          Z= ej>        e?          Z@deAfdZBd\deCdeDfdZE e1d           G d  d!e jF                              ZGe1 G d" d#eH                      ZIe1d$eeDeee         ejJ        f         f         dejK        fd%            ZL e1d&          d'eee         ejM        e'f         d(eDdejJ        fd)            ZNd'eee         ejM        f         d(eDdejJ        fd*ZOd'eeejJ        ejP        f                  fd+ZQd'ejM        d,ejR        d(eDfd-ZSd'eee         ejM        f         deejT                 fd.ZUd/d/d/d/d/d/d/d0d1d2d2d2d2d3ZVd'eee         ejM        f         deejT                 fd4ZWe1d5             ZXe1d6             ZYe1d7             ZZ G d8 d9eGej[                  Z\ e2d           G d: d;e\                      Z] e2d&           G d< d=e\                      Z^ e2d           G d> d?ej_                              Z` e2d           G d@ dAeja                              Zb e2d&           G dB dCeGej[                              Zc e2d&           G dD dEeja                              ZddFeeedGf         dHeedeeedGf         fdIZfee]e^ecf         Zg e1d&          dJeeg         degfdK            Zh e1d&          dLeeebedf                  deeebedf                  fdM            Zi e1d&          	 d]dOeedP                  dQeAdedP         fdR            ZjdLeejM        eejM                 f         dejM        fdSZkdLeejM        eejM                 f         deAfdTZldUejM        deefdVZmdUejM        dejM        fdWZndXejM        deefdYZodZ Zp	  ejq         e]d[ ejr                                          ejq         e^d[ ejr                                          ejq         ec ejr                    d                      dS # ejs        $ r Y dS w xY w)^    N)abstractmethod)datetime)Enum)Any
CollectionDictIterableListOptionalTupleUnion)parse)_check_pyarrow_versionget_pyarrow_version)env_integer)#MIN_PYARROW_VERSION_SCALAR_SUBCLASSArrowPythonObjectArray_object_extension_type_allowed)	ArrayLike"_is_ndarray_variable_shaped_tensor_should_convert_to_tensorcreate_ragged_ndarray) _convert_datetime_to_np_datetimeconvert_to_numpy)log_once)DeveloperAPI	PublicAPI)	INT32_MAXz13.0.0z12.0.0   c                       e Zd ZdZdZdS )_SerializationFormatr      N)__name__
__module____qualname__JSONCLOUDPICKLE     x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/air/util/tensor_extensions/arrow.pyr!   r!   7   s        DKKKr)   r!   9RAY_DATA_ARROW_EXTENSION_SERIALIZATION_LEGACY_JSON_FORMATr"   )RAY_EXTENSION_SERIALIZATION_CACHE_MAXSIZEi returnc                  0    t                      t          k    S N)r   .MIN_PYARROW_VERSION_EXT_ARRAY_CONCAT_SUPPORTEDr(   r)   r*   !_extension_array_concat_supportedr1   M   s      $RRRr)   data
serialized
field_namec           
          	 t          j        |           S # t          $ rM 	 t          j        |           cY S # t          j        $ r# t          d| dt          |                      w xY ww xY w)z:Deserialize data with cloudpickle first, fallback to JSON.zUnable to deserialize z from )cloudpickleloads	ExceptionjsonJSONDecodeError
ValueErrortype)r3   r4   s     r*   _deserialize_with_fallbackr=   Q   s    
 ,,,   	:j)))))# 	 	 	MMM4
;K;KMM  		s    
A-7A-2A))A-beta)	stabilityc            
       *    e Zd ZdZdededdf fdZdefdZedefd            Z	e
 ej        e	          d
edededefd                        Ze
edededefd                        Ze
edefd                        Ze
defd            Z xZS )'ArrowExtensionSerializeDeserializeCachea  Base class for caching Arrow extension type serialization and deserialization.

    The deserialization and serialization of Arrow extension types is frequent,
    so we cache the results here to improve performance.

    The deserialization cache uses functools.lru_cache as a classmethod. There is
    a single cache instance shared across all subclasses, but the cache key includes
    the class (cls parameter) as the first argument, so different subclasses get
    different cache entries even when called with the same parameters. The cache is
    thread-safe and has a maximum size limit to control memory usage. The cache key
    is (cls, *args) where args are the parameters returned by _get_deserialize_parameter().

    Attributes:
        _serialize_cache: Instance-level cache for serialization results.
            This is a simple cached value (bytes) that is computed once per
            instance and reused.
    argskwargsr-   Nc                 x    d| _         t          j                    | _         t	                      j        |i | dS )zInitialize the extension type with caching support.

        Args:
            *args: Positional arguments passed to the parent class.
            **kwargs: Keyword arguments passed to the parent class.
        N)_serialize_cache	threadingRLock_cache_locksuper__init__)selfrB   rC   	__class__s      r*   rJ   z0ArrowExtensionSerializeDeserializeCache.__init__t   s?     !%$?,,$)&)))))r)   c                     | j         | j         S | j        5  | j         |                                 | _         | j         cddd           S # 1 swxY w Y   dS )z6Serialize the extension type using caching if enabled.N)rE   rH   _arrow_ext_serialize_computerK   s    r*   __arrow_ext_serialize__z?ArrowExtensionSerializeDeserializeCache.__arrow_ext_serialize__   s     ,(( 	) 	)$,(,(I(I(K(K%(	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	)s   'A

AAc                     dS )z?Subclasses must implement this method to compute serialization.Nr(   rO   s    r*   rN   zDArrowExtensionSerializeDeserializeCache._arrow_ext_serialize_compute   s	     	r)   )maxsizeclsc                      | j         |i |S )a;  Deserialize the extension type using the class-level cache.

        This method is cached using functools.lru_cache to improve performance
        when deserializing extension types. The cache key includes the class (cls)
        as the first argument, ensuring different subclasses get separate cache entries.

        Args:
            *args: Positional arguments passed to _arrow_ext_deserialize_compute.
            **kwargs: Keyword arguments passed to _arrow_ext_deserialize_compute.

        Returns:
            The deserialized extension type instance.
        )_arrow_ext_deserialize_computerS   rB   rC   s      r*   _arrow_ext_deserialize_cachezDArrowExtensionSerializeDeserializeCache._arrow_ext_deserialize_cache   s      2s14B6BBBr)   c                     dS )zASubclasses must implement this method to compute deserialization.Nr(   rV   s      r*   rU   zFArrowExtensionSerializeDeserializeCache._arrow_ext_deserialize_compute   	     	r)   c                     dS )z]Subclasses must implement this method to return the parameters for the deserialization cache.Nr(   rS   storage_typer3   s      r*   _get_deserialize_parameterzBArrowExtensionSerializeDeserializeCache._get_deserialize_parameter   rY   r)   c                 >     | j         |                     ||           S )z8Deserialize the extension type using caching if enabled.)rW   r]   r[   s      r*   __arrow_ext_deserialize__zAArrowExtensionSerializeDeserializeCache.__arrow_ext_deserialize__   s)     0s/++L*EE
 	
r)   )r#   r$   r%   __doc__r   rJ   bytesrP   r   rN   classmethod	functools	lru_cache+ARROW_EXTENSION_SERIALIZATION_CACHE_MAXSIZEr<   rW   rU   r   r]   r_   __classcell__rL   s   @r*   rA   rA   `   s        $
*c 
*S 
*T 
* 
* 
* 
* 
* 
*) ) ) ) ) e    ^ Y!LMMMC$ Cs Cc Cc C C C NM [C  3 # #    ^ [ U    ^ [ 
C 
 
 
 [
 
 
 
 
r)   rA   c                   ,     e Zd ZdZdZdef fdZ xZS )ArrowConversionErrorz=Error raised when there is an issue converting data to Arrow.   data_strc                     t          |          | j        k    r|d | j                 dz   }d| }t                                          |           d S )Nz...z Error converting data to Arrow: )lenMAX_DATA_STR_LENrI   rJ   )rK   rk   messagerL   s      r*   rJ   zArrowConversionError.__init__   sZ    x==4000 7$"7 785@H?X??!!!!!r)   )r#   r$   r%   r`   rn   strrJ   rf   rg   s   @r*   ri   ri      sP        GG" " " " " " " " " " "r)   ri   pydictc                     	 t           j                            |           S # t          $ r"}t	          t          |                     |d}~ww xY w)z}
    Convert a Python dictionary to a pyarrow Table.

    Raises:
        ArrowConversionError: if the conversion fails.
    N)paTablefrom_pydictr8   ri   rp   )rq   es     r*   pyarrow_table_from_pydictrw      sR    7x##F+++ 7 7 7"3v;;//Q67s   ! 
AAAalphacolumn_valuescolumn_namec           	      (   	 t          |           dk    r9t          | |          r)ddlm} |                    t          |           |          S t          | |          S # t          $ r}ddlm	} |
                                j        }t                      sd}dt           dt           d}n|du p|}|rd	}nd
}||s4t          d          r%t                               d| d| d| |           |s t%          j        |           cY d}~S d}~ww xY w)a  Converts provided NumPy `ndarray` into PyArrow's `array` while utilizing
    both Arrow's natively supported types as well as custom extension types:

        - ArrowTensorArray (for tensors)
        - ArrowPythonObjectArray (for user-defined python class objects, as well as
        any python object that aren't represented by a corresponding Arrow's native
        scalar type)
    r   ArrowTensorArrayDataContextFz[skipping fallback to serialize as pickled python objects (due to unsupported Arrow version z, min required version is )Nz3falling back to serialize as pickled python objectsz~skipping fallback to serialize as pickled python objects (due to DataContext.enable_fallback_to_arrow_object_ext_type = False)0_fallback_to_arrow_object_extension_type_warningzFailed to convert column 'z' into pyarrow array due to: z; exc_info)rm   r   $ray.data.extensions.tensor_extensionr}   
from_numpyr    _convert_to_pyarrow_native_arrayri   ray.data.contextr   get_current(enable_fallback_to_arrow_object_ext_typer   PYARROW_VERSIONr   r   loggerwarningr   from_objects)ry   rz   r}   acer   enable_fallback_config object_ext_type_fallback_allowedobject_ext_type_details           r*   convert_to_pyarrow_arrayr      s   FB }!!&?;'
 '
! NMMMMM $.. //   4M;OOO 5B 5B 5B000000 ##%%N 	 .// 	/4,R>MR R+NR R R #" '$.H2H - 0 	I '&
 ' #*2R*IJJ +NNA[ A A!$A A(>A A     0 	 &2=AAAAAAAAk5Bs%   AA A 
D(BDDDc                 @   	 t          |           dk    r*t          | d         t                    rt          |           } t          |           dk    r;t          | d         t          j        t          j        f          rt          |           S t          |           }|rJt          j	        
                    |          r+t          | t          j                  rt          | ||          } t                              t!          j        d          d| d| d           t          |           dk    r,t          | d         t          j                  rd | D             } t	          j        | |          S # t(          $ r"}t+          t-          |                     |d}~ww xY w)	zConverts provided NumPy `ndarray` into PyArrow's `array` while only utilizing
    Arrow's natively supported types (ie no custom extension types)r   TRACEzInferred dtype of 'z' for column ''c                 $    g | ]}|j         r|nd S r/   )is_valid.0vs     r*   
<listcomp>z4_convert_to_pyarrow_native_array.<locals>.<listcomp>W  s#    NNN1!*6QQ$NNNr)   r<   N)rm   
isinstancer   r   rs   ArrayChunkedArray_combine_as_list_array_infer_pyarrow_typetypesis_timestampnpndarray-_coerce_np_datetime_to_pa_timestamp_precisionr   loglogginggetLevelName
ListScalararrayr8   ri   rp   )ry   rz   pa_typerv   s       r*   r   r   '  s   .>
 }!!jq1A8&L&L!<]KKM }!!j!rx9'
 '
! *-888 &m44 	rx,,W55 	
 -44  M!7K! ! 	

 ))G'GGGGG	
 	
 	
 }!!jq1A2=&Q&Q!NNNNNMxG4444 > > >"3}#5#566A=>s   B
E1 C#E1 1
F;FFc                 z   d | D             }t          j        t          j        dgt          j        |          g          t          j                              }t          j        t          j        d | D                        }t           j	        
                    ||t          j        |j                            S )z7Combines list of Arrow arrays into a single `ListArray`c                 ,    g | ]}t          |          S r(   rm   r   s     r*   r   z*_combine_as_list_array.<locals>.<listcomp>b  s    ***qCFF***r)   r   r   c                 V    g | ]&}t          |t          j                  r|j        n|g'S r(   )r   rs   r   chunksr   s     r*   r   z*_combine_as_list_array.<locals>.<listcomp>h  sA        'q"/::C  r)   )rs   r   r   concatenatecumsumint32concat_arrays	itertoolschain	ListArrayfrom_arrayslist_r<   )ry   lensoffsetscombineds       r*   r   r   ^  s     +*M***Dhr~sBIdOO&<==BHJJOOOG  &  	
 H <##GXrx7N7NOOOr)   dtypec                 >   t          j        | j        t           j                  sJ t          j        | j                  \  }}|j        }||k    rM|                     d| d          } t          d| d          r!t          	                    d|d| d           | S )Nzdatetime64[]column__timestamp_warningzConverting a z$ precision datetime NumPy array to 'z' precision Arrow timestamp. This conversion occurs because Arrow supports fewer precisions than Arrow and might result in a loss of precision or unrepresentable values.)
r   
issubdtyper   
datetime64datetime_dataunitastyper   r   r   )ry   r   rz   numpy_precision_arrow_precisions         r*   r   r   s  s     =,bm<<<<<)-*=>>OQjO/)) &,,-M?-M-M-MNN=k===>> 	NN* * *,* * *   r)   c                    t          |           dk    rdS t          |           }||S t          j        |           }dt          dt
          fdt          j                            |          r.t          fd| D                       rt          j	                    S t          j        
                    |          r.t          fd| D                       rt          j                    S |S )a  Infers target Pyarrow `DataType` based on the provided
    columnar values.

    NOTE: This is a wrapper on top of `pa.infer_type(...)` utility
          performing up-casting of `binary` and `string` types to
          corresponding `large_binary` and `large_string` types in case
          any of the array elements exceeds 2Gb in size therefore
          making it impossible for original types to accommodate such
          values.

          Unfortunately, for unknown reasons PA doesn't perform
          that upcasting itself henceforth we have to do perform
          it manually

    Args:
        column_values: List of columnar values

    Returns:
        Instance of PyArrow's `DataType` based on the provided
        column values
    r   Nobjr-   c                 n    t          | t          t          f          rt          |           t          k    S dS )NF)r   rp   ra   rm   r   )r   s    r*   _len_gt_overflow_thresholdz7_infer_pyarrow_type.<locals>._len_gt_overflow_threshold  s0     cC<(( 	(s88i''ur)   c                 &    g | ]} |          S r(   r(   r   r   r   s     r*   r   z'_infer_pyarrow_type.<locals>.<listcomp>  %    >>>1	#	#A	&	&>>>r)   c                 &    g | ]} |          S r(   r(   r   s     r*   r   z'_infer_pyarrow_type.<locals>.<listcomp>  r   r)   )rm   _try_infer_pa_timestamp_typers   
infer_typer   boolr   	is_binaryanylarge_binary	is_stringlarge_string)ry   dtype_with_timestamp_typeinferred_pa_dtyper   s      @r*   r   r     s   2 =Qt
 !=] K K ,((m44      
x+,, !>>>>>>>2 2 !    			-	.	. !3>>>>>>>4 4 !    r)   smsusns)YDMWhmr   r   r   r   psfsasc                    t          | t                    r@t          |           dk    r-t          | d                   }|rt	          j        |          nd S t          | t          j                  r_t          j        | j	        t          j
                  r;t          j        | j	                  \  }}t	          j        t          |                   S d S )Nr   )r   listrm   r   rs   r   r   r   r   r   r   r   	timestamp_NUMPY_TO_ARROW_PRECISION_MAP)ry   element_typenp_precisionr   s       r*   r   r     s     -&& @3}+=+=+A+A 4M!4DEE)5?rx%%%4?-,, R]2 2  *=+>??a|9,GHHH tr)   c                  @    g t                      t                      R S )zXReturns list of extension types of Arrow Array holding
    multidimensional tensors
    ),get_arrow_extension_fixed_shape_tensor_types/get_arrow_extension_variable_shape_tensor_typesr(   r)   r*    get_arrow_extension_tensor_typesr     s-    
	5	7	7	8	:	:  r)   c                      t           t          fS z`Returns list of Arrow extension types holding multidimensional
    tensors of *fixed* shape
    )ArrowTensorTypeArrowTensorTypeV2r(   r)   r*   r   r     s    
 ---r)   c                      t           fS r   )ArrowVariableShapedTensorTyper(   r)   r*   r   r     s    
 *++r)   c                       e Zd ZdZdeedf         dej        def fdZ	e
deedf         fd            Ze
dej        fd	            Zd
 Zd Zd Zd Zd Zdddej        fdZdefdZdefdZd Zd ZdefdZ xZS )_BaseFixedShapeArrowTensorTypea  
    Arrow ExtensionType for an array of fixed-shaped, homogeneous-typed
    tensors.

    This is the Arrow side of TensorDtype.

    See Arrow extension type docs:
    https://arrow.apache.org/docs/python/extending_types.html#defining-extension-types-user-defined-types
    shape.tensor_dtypeext_type_idc                 Z    || _         t                                          ||           d S r/   )_shaperI   rJ   )rK   r   r   r   rL   s       r*   rJ   z'_BaseFixedShapeArrowTensorType.__init__  s,     {33333r)   r-   c                     | j         S )z-
        Shape of contained tensors.
        )r   rO   s    r*   r   z$_BaseFixedShapeArrowTensorType.shape"  s    
 {r)   c                     | j         j        S )3Returns the type of the underlying tensor elements.)r\   
value_typerO   s    r*   scalar_typez*_BaseFixedShapeArrowTensorType.scalar_type)  s      ++r)   c                 ^    ddl m}  || j        | j                                                  S )
        Convert Arrow extension type to corresponding Pandas dtype.

        Returns:
            An instance of pd.api.extensions.ExtensionDtype.
        r   TensorDtype)%ray.air.util.tensor_extensions.pandasr	  r   r  to_pandas_dtyperK   r	  s     r*   r  z._BaseFixedShapeArrowTensorType.to_pandas_dtype.  s:     	FEEEEE{4;(8(H(H(J(JKKKr)   c                 F    | j         | j        |                                 ffS r/   r_   r\   rP   rO   s    r*   
__reduce__z)_BaseFixedShapeArrowTensorType.__reduce__9  ,    -((**0
 
 	
r)   c                    t           t          j        k    rt          j        | j                  S t           t          j        k    r+t          j        | j                                                  S t          dt                      NzInvalid serialization format: )
$ARROW_EXTENSION_SERIALIZATION_FORMATr!   r'   r6   dumpsr   r&   r9   encoder;   rO   s    r*   rN   z;_BaseFixedShapeArrowTensorType._arrow_ext_serialize_compute?  so    /3G3SSS$T[11115I5NNN:dk**11333W1UWW  r)   c                     t           S z
        ExtensionArray subclass with custom logic for this array of tensors
        type.

        Returns:
            A subclass of pd.api.extensions.ExtensionArray.
        r|   rO   s    r*   __arrow_ext_class__z2_BaseFixedShapeArrowTensorType.__arrow_ext_class__I  s
      r)   c                     t           S z\
        ExtensionScalar subclass with custom logic for this array of tensors type.
        ArrowTensorScalarrO   s    r*   __arrow_ext_scalar_class__z9_BaseFixedShapeArrowTensorType.__arrow_ext_scalar_class__S  
     ! r)   scalarpa.ExtensionScalarc                     |j         dS |j         j        }|j        j        }|j        }|j        }|                                d         }t          ||||          S )A
        Convert an ExtensionScalar to a tensor element.
        Nr"   )valuevaluesr<   r   offsetbuffers_to_ndarray_helper)rK   r  
raw_valuesr   r  r%  data_buffers          r*   _extension_scalar_to_ndarrayz;_BaseFixedShapeArrowTensorType._extension_scalar_to_ndarrayY  s^    
 <4\(
!_
" ((**1-!%V[IIIr)   c                 J    | j         j         d| j         d| j        j         dS )Nz(shape=, dtype=r   )rL   r#   r   r\   r  rO   s    r*   __str__z&_BaseFixedShapeArrowTensorType.__str__h  s.    .)ee$*eedFWFbeeeer)   c                      t          |           S r/   rp   rO   s    r*   __repr__z'_BaseFixedShapeArrowTensorType.__repr__k      4yyr)   c                     t          |t          |                     o/|j        | j        k    o|j        | j        k    o|j        | j        k    S r/   )r   r<   extension_namer   r  rK   others     r*   __eq__z%_BaseFixedShapeArrowTensorType.__eq__n  sS    ud4jj)) 6$(;;6tz)6 !T%55		
r)   c                 .    |                      |           S r/   r6  r4  s     r*   __ne__z%_BaseFixedShapeArrowTensorType.__ne__v      ;;u%%%%r)   c                 D    t          | j        | j        | j        f          S r/   )hashr3  r  r   rO   s    r*   __hash__z'_BaseFixedShapeArrowTensorType.__hash__z  s    T($*:DKHIIIr)   )r#   r$   r%   r`   r   intrs   DataTyperp   rJ   propertyr   r  r  r  rN   r  r  r   r   r*  r-  r0  r6  r9  r=  rf   rg   s   @r*   r   r     s        438_446K4NQ4 4 4 4 4 4 uS#X    X ,R[ , , , X,	L 	L 	L
 
 
       ! ! !J3G JBJ J J J Jf f f f f#    
 
 
& & &J# J J J J J J J Jr)   r   c                        e Zd ZdZ ej                    Zdeedf         dej	        f fdZ
ed             Zed             Z xZS )r   zArrow ExtensionType (v1) for tensors.

    NOTE: This type does *NOT* support tensors larger than 4Gb (due to
          overflow of int32 offsets utilized inside Pyarrow `ListType`)
    r   .r   c                 r    t                                          |t          j        |          d           dS )
        Construct the Arrow extension type for array of fixed-shaped tensors.

        Args:
            shape: Shape of contained tensors.
            dtype: pyarrow dtype of tensor elements.
        zray.data.arrow_tensorN)rI   rJ   rs   r   rK   r   r   rL   s      r*   rJ   zArrowTensorType.__init__  s0     	1HIIIIIr)   c                     ||j         fS r/   r  r[   s      r*   r]   z*ArrowTensorType._get_deserialize_parameter      L344r)   c                 T    t          t          |d                    } | ||          S Nr   tupler=   rS   r3   r  r   s       r*   rU   z.ArrowTensorType._arrow_ext_deserialize_compute  ,    0WEEFFs5*%%%r)   )r#   r$   r%   r`   rs   r   OFFSET_DTYPEr   r>  r?  rJ   rb   r]   rU   rf   rg   s   @r*   r   r   ~  s          28::L	JeCHo 	Jbk 	J 	J 	J 	J 	J 	J 5 5 [5 & & [& & & & &r)   r   c                        e Zd ZdZ ej                    Zdeedf         dej	        f fdZ
ed             Zed             Z xZS )r   z@Arrow ExtensionType (v2) for tensors (supporting tensors > 4Gb).r   .r   c                 r    t                                          |t          j        |          d           dS )rC  zray.data.arrow_tensor_v2N)rI   rJ   rs   
large_listrD  s      r*   rJ   zArrowTensorTypeV2.__init__  s2     	e 4 46PQQQQQr)   c                     ||j         fS r/   rF  r[   s      r*   r]   z,ArrowTensorTypeV2._get_deserialize_parameter  rG  r)   c                 T    t          t          |d                    } | ||          S rI  rJ  rL  s       r*   rU   z0ArrowTensorTypeV2._arrow_ext_deserialize_compute  rM  r)   )r#   r$   r%   r`   rs   int64rN  r   r>  r?  rJ   rb   r]   rU   rf   rg   s   @r*   r   r     s        JJ28::L	ReCHo 	Rbk 	R 	R 	R 	R 	R 	R 5 5 [5 & & [& & & & &r)   r   c                   :    e Zd Zdej        fdZdej        fdZdS )r  r-   c                 *    |                                  S r/   )	__array__)rK   rC   s     r*   as_pyzArrowTensorScalar.as_py  s    ~~r)   c                 6    | j                             |           S r/   )r<   r*  rO   s    r*   rW  zArrowTensorScalar.__array__  s    y55d;;;r)   N)r#   r$   r%   r   r   rX  rW  r(   r)   r*   r  r    sN                 <2: < < < < < <r)   r  c            	           e Zd ZdZe	 ddeej        eej                 f         de	e
         ded         fd            Zedej        ded         fd            Zdd
efdZdeddfdZdS )r}   z
    An array of fixed-shape, homogeneous-typed tensors.

    This is the Arrow side of TensorArray.

    See Arrow docs for customizing extension arrays:
    https://arrow.apache.org/docs/python/extending_types.html#custom-extension-array-class
    Narrrz   r-   r}   ArrowVariableShapedTensorArrayc                 D   t          |t          j                  s$t          |t                    rt	          |          }t          |t          t
          f          r|rt          |d         t          j                  rj	 t          j        |d          }nR# t          $ rE}t          	                    d| d|           t          j
        |t                    }Y d}~nd}~ww xY wt          |t          j                  s"t          dt          |           d	|           	 t          |          }|rt          |||          }|                     |          S # t           $ r:}d
}|r	|d| dz  }|d|j         d|j         d| z  }t'          |          |d}~ww xY w)aN  
        Convert an ndarray or an iterable of ndarrays to an array of homogeneous-typed
        tensors. If given fixed-shape tensor elements, this will return an
        ``ArrowTensorArray``; if given variable-shape tensor elements, this will return
        an ``ArrowVariableShapedTensorArray``.

        Args:
            arr: An ndarray or an iterable of ndarrays.
            column_name: Optional. Used only in logging outputs to provide
                additional details.

        Returns:
            - If fixed-shape tensor elements, an ``ArrowTensorArray`` containing
              ``len(arr)`` tensors of fixed shape.
            - If variable-shaped tensor elements, an ``ArrowVariableShapedTensorArray``
              containing ``len(arr)`` tensors of variable shape.
            - If scalar elements, a ``pyarrow.Array``.
        r   )axiszFailed to stack lists due to: z3; falling back to using np.array(..., dtype=object)r   r   Nz/Must give ndarray or iterable of ndarrays, got   z	column: 'z', zshape: z	, dtype: z, data: )r   r   r   r	   r   rK  stackr;   r   r   r   objectr<   r   r   _from_numpyr8   r   r   ri   )rS   r[  rz   vetimestamp_dtyperv   rk   s          r*   r   zArrowTensorArray.from_numpy  s   0 #rz** 	z#x/H/H 	s))CcD%=)) 	2c 	2jQ6T6T 	2
2hs+++ 2 2 2IR I I I     hs&1112 #rz** 	S$s))SScSS  	8:3??O 
 D+  ??3''' 	8 	8 	8H 988888N#)NNciNNNNNH&x00a7	8s0   >B 
C$;CC$$6E 
F%5FFc                 t   t          |          dk    r.t          j        |d                   rt          j        |          S t          |          rt                              |          S |j        j	        st          j
        |          }t          j        |j                  }t          j                            |          re|j        j        dk    s |j        j        dk    r't           j        dk    rt#          d|j                   t          j        |j        j                  }|j        d         }|j        dd          }|j        }|rt          j        |          nd}t          j                            |          rt          j        |d          }t          j        |          }t          j                            ||d |g          }dd	lm}	 |	                                j        rtA          ||          }
ntC          ||          }
|
j"        #                                }|dk    rt          j$        |dz   |
          }nt          j%        d|dz   |z  ||
          }t          j        |          }t          j                            |
j&        |d |g|g          }|
'                    |          S )Nr   >=big:Only little-endian string tensors are supported, but got: r"   littlebitorderr~   r`  )children)(rm   r   isscalarrs   r   r   r]  r   flagsc_contiguousascontiguousarrayfrom_numpy_dtyper   r   r   	byteordersysr;   binaryitemsizer   sizeprod
is_booleanpackbits	py_bufferr   from_buffersray.datar   r   use_arrow_tensor_v2r   r   rN  r  zerosaranger\   
wrap_array)rS   r[  scalar_dtype	outer_lenelement_shapetotal_num_itemsnum_items_per_elementr)  
data_arrayr   pa_type_offset_dtyper   offset_bufferstorages                  r*   re  zArrowTensorArray._from_numpy  s   
 s88a<<BKA//<8C== -c22 	B 2<<SAAAy% 	,&s++C*39558l++ 	9y"c))	#s**s}/E/E , #	, ,   9SY%788LIaL		!""(:G N 6 6 6Q 8|,, 	6
 +cH555Cl3''X**/D++>
 

 	)(((((""$$8 	D(EEHH&}lCCH,<<>> !A%%hy1}LAAAGGiQ"77%"	  G W--(''!=! \	 ( 
 
 ""7+++r)   Tzero_copy_onlyc                 @   |                                  }|d         }| j        j        }|j        }| j        j        }t
          j                            |          }|r|j        }n
|j        dz  }|rt          j
        |          nd}	| j        |	z  }
||
z  }t          |           f|z   }|r|dz  }|dz  }d|t          j
        |          z   dz
  dz  z   }t          j        |ft          j        ||          }t          j        |d          }t          j        |t          j        ||          S t
          j                            |          r%t          j        d|j        t(          z             }n|                                }t          j        ||||          S )a  
        Convert the entire array of tensors into a single ndarray.

        Args:
            zero_copy_only: If True, an exception will be raised if the
                conversion to a NumPy array would require copying the
                underlying data (e.g. in presence of nulls, or for
                non-primitive types). This argument is currently ignored, so
                zero-copy isn't enforced even if this argument is true.

        Returns:
            A single ndarray representing the entire array of tensors.
              r"   r   bufferr%  rm  rn  <U)r&  r  r<   r  r   rs   r   r|  	bit_widthr   r{  r%  rm   r   uint8
unpackbitsbool_is_fixed_size_binaryr   
byte_widthNUM_BYTES_PER_UNICODE_CHARr  )rK   r  r&  r)  storage_list_typer  r   r|  buffer_item_widthr  buffer_offsetr%  byte_bucket_offsetbool_offsetnum_boolean_byte_bucketsr[  	ext_dtypes                    r*   to_numpyzArrowTensorArray.to_numpy\  s     ,,..aj L-&1
	 X((44
  	: !+ 4 !+ 4 9 38 >Q&;;"]2Tu$ 	U
 "(1 1*K ()[275>>-IA-MRS,S'T$*)+h")	  C -h777C:e28CTTTT 8((44 	5JZ*.HHJJ II #2244Iz%yVTTTTr)   ndimr]  c                 \   | j         j        }|t          |          k     rt          d| j          d|          t	          | j         t
                    s,t                              |                                           S t          ||          }t          j        t          | j                  t          j                                      |          }t          j                            | j        |gddg          }t#          | j         j        |          }|                    |          S )zN
        Convert this tensor array to a variable-shaped tensor array.
        Can't convert % to var-shaped tensor type with ndim=r   r2   r   r  )r<   r   rm   r;   r   r   r]  r   r  _pad_shape_with_singleton_axesrs   nullsr  SHAPES_ARRAY_TYPE	fill_nullStructArrayr   r   r  r  )rK   r  r   target_shapeshape_arrayr  target_types          r*   to_var_shaped_tensor_arrayz+ArrowTensorArray.to_var_shaped_tensor_array  s'    	#e**SSSDSS   $)%677 	N1<<T]]__MMM
 6eTBBh/A
 
 
 )L
!
! 	
 .,,\;'W
 

 4I!
 
 

 %%g...r)   r/   T)r#   r$   r%   r`   rb   r   r   r   r	   r   rp   r   re  r   r  r>  r  r(   r)   r*   r}   r}     s          &*?8 ?82:x
334?8 c]?8 
C	D	?8 ?8 ?8 [?8B K,ZK, 
C	DK, K, K, [K,ZLU LUt LU LU LU LU\*/*/ 
**/ */ */ */ */ */r)   r}   c                   R    e Zd ZdZ ej                    Zdej        def fdZ	d Z
edefd            Zeded         fd	            Zedej        fd
            Zd Zd Zed             Zed             Zd Zd ZdefdZdefdZd Zd ZdefdZdddej        fdZ xZ S )r   ad  
    Arrow ExtensionType for an array of heterogeneous-shaped, homogeneous-typed
    tensors.

    This is the Arrow side of ``TensorDtype`` for tensor elements with different shapes.

    NOTE: This extension only supports tensor elements with non-ragged, well-defined
    shapes; i.e. every tensor element must have a well-defined shape and all of their
    shapes have to have same number of dimensions (ie ``len(shape)`` has to be the
    same for all of them).

    See Arrow extension type docs:
    https://arrow.apache.org/docs/python/extending_types.html#defining-extension-types-user-defined-types
    r   r  c           	          || _         t                                          t          j        dt          j        |          fdt          j        | j                  fg          d           dS )z
        Construct the Arrow extension type for array of heterogeneous-shaped tensors.

        Args:
            dtype: pyarrow dtype of tensor elements.
            ndim: The number of dimensions in the tensor elements.
        r2   r   z%ray.data.arrow_variable_shaped_tensorN)_ndimrI   rJ   rs   structrQ  r   rN  )rK   r   r  rL   s      r*   rJ   z&ArrowVariableShapedTensorType.__init__  so     
I"-../'28DDU;V;V1WX  4		
 	
 	
 	
 	
r)   c                 ~    ddl m}  || j        | j        d         j        j                                                  S )r  r   r  r2   )r
  r	  r   r\   r<   r  r  r  s     r*   r  z-ArrowVariableShapedTensorType.to_pandas_dtype  sM     	FEEEEE{Jf%*5EEGG
 
 	
r)   r-   c                     | j         S )z7Return the number of dimensions in the tensor elements.)r  rO   s    r*   r  z"ArrowVariableShapedTensorType.ndim  s     zr)   )N.c                     d| j         z  S )Nr/   r  rO   s    r*   r   z#ArrowVariableShapedTensorType.shape  s    ""r)   c                 d    | j                             d          }| j         |         j        j        S )r  r2   )r\   get_field_indexr<   r  )rK   data_field_indexs     r*   r  z)ArrowVariableShapedTensorType.scalar_type  s0      ,<<VDD !127BBr)   c                 F    | j         | j        |                                 ffS r/   r  rO   s    r*   r  z(ArrowVariableShapedTensorType.__reduce__  r  r)   c                    t           t          j        k    rt          j        | j                  S t           t          j        k    r+t          j        | j                                                  S t          dt                      r  )
r  r!   r'   r6   r  r  r&   r9   r  r;   rO   s    r*   rN   z:ArrowVariableShapedTensorType._arrow_ext_serialize_compute   so    /3G3SSS$TZ00015I5NNN:dj))00222W1UWW  r)   c                 *    ||d         j         j        fS )Nr2   )r<   r  r[   s      r*   r]   z8ArrowVariableShapedTensorType._get_deserialize_parameter*  s    L05@AAr)   c                 :    t          |d          } | ||          S )Nr  )r=   )rS   r3   r  r  s       r*   rU   z<ArrowVariableShapedTensorType._arrow_ext_deserialize_compute.  s#    )*f==s:t$$$r)   c                     t           S r  )r]  rO   s    r*   r  z1ArrowVariableShapedTensorType.__arrow_ext_class__3  s
     .-r)   c                     t           S r  r  rO   s    r*   r  z8ArrowVariableShapedTensorType.__arrow_ext_scalar_class__=  r  r)   c                 L    | j         d         j        j        }d| j         d| dS )Nr2   z#ArrowVariableShapedTensorType(ndim=r,  r   )r\   r<   r  r  )rK   r   s     r*   r-  z%ArrowVariableShapedTensorType.__str__C  s0    !&).9PTYPPPPPPr)   c                      t          |           S r/   r/  rO   s    r*   r0  z&ArrowVariableShapedTensorType.__repr__G  r1  r)   c                 l    t          |t                    o|j        | j        k    o|j        | j        k    S r/   )r   r   r3  r  r4  s     r*   r6  z$ArrowVariableShapedTensorType.__eq__J  s=     u;<< 6$(;;6!T%55	
r)   c                 .    |                      |           S r/   r8  r4  s     r*   r9  z$ArrowVariableShapedTensorType.__ne__S  r:  r)   c                 8    t          | j        | j        f          S r/   )r<  r3  r  rO   s    r*   r=  z&ArrowVariableShapedTensorType.__hash__W  s    T($*:;<<<r)   r  r   c                 <   |j         dS |j                             d          }|j        }|j        }|j        }|                                d         }t          |j                             d                                                    }t          ||||          S )r"  Nr2   r"   r   )	r#  getr$  r<   r%  r&  rK  rX  r'  )rK   r  r2   r(  r  r%  r)  r   s           r*   r*  z:ArrowVariableShapedTensorType._extension_scalar_to_ndarrayZ  s     <4|''[
_
" ((**1-fl&&w//557788!%V[IIIr)   )!r#   r$   r%   r`   rs   rT  rN  r?  r>  rJ   r  r@  r  r   r   r  r  rN   rb   r]   rU   r  r  rp   r-  r0  r6  r9  r=  r   r   r*  rf   rg   s   @r*   r   r     s         28::L
bk 
 
 
 
 
 
 
 
 
 
 c    X #uY' # # # X# CR[ C C C XC

 
 
   B B [B % % [%. . .! ! !Q Q Q Q Q#    
 
 
& & &=# = = = =J3G JBJ J J J J J J J Jr)   r   c                       e Zd ZdZ ej         ej                              Zede	e
j        ee
j                 ee
j                 f         dd fd            ZddefdZdedd fd	Zd
S )r]  aB  
    An array of heterogeneous-shaped, homogeneous-typed tensors.

    This is the Arrow side of TensorArray for tensor elements that have differing
    shapes. Note that this extension only supports non-ragged tensor elements; i.e.,
    when considering each tensor element in isolation, they must have a well-defined
    shape. This extension also only supports tensor elements that all have the same
    number of dimensions.

    See Arrow docs for customizing extension arrays:
    https://arrow.apache.org/docs/python/extending_types.html#custom-extension-array-class
    r[  r-   c                 |   t          |t          t          t          j        f          st          dt          |                     t          |          dk    rt          d          t          j        t          |          t          j	                  }t          j        t          |          t          j	                  }t          j
        t          |          t          j                  }d}t          |          D ]t\  }}t          j        |          }|%|j        |k    rt          d| d|j                   |j        }|j        ||<   |j        ||<   t          j        |d	          ||<   ut          j        |          }|d
         }	t'          |          }
|
j        }t+          j        |          }t*          j                            |          rQ|j        dk    s|j        dk    r"t4          j        dk    rt          d|           t+          j        |j                  }|j        t          j        u r!|
j        dk    rt          j        |
d          }
t+          j        |
          }
t*          j         !                    ||	d|
g          }t          j"        |dd          }t+          j#        |          }t*          j$        %                    ||          }t+          j#        |          }t*          j&        %                    ||gddg          }tO          ||          }|(                    |          S )ax  
        Convert an ndarray or an iterable of heterogeneous-shaped ndarrays to an array
        of heterogeneous-shaped, homogeneous-typed tensors.

        Args:
            arr: An ndarray or an iterable of heterogeneous-shaped ndarrays.

        Returns:
            An ArrowVariableShapedTensorArray containing len(arr) tensors of
            heterogeneous shape.
        zmArrowVariableShapedTensorArray can only be constructed from an ndarray or a list/tuple of ndarrays, but got: r   z5Creating empty ragged tensor arrays is not supported.r`  NzArrowVariableShapedTensorArray only supports tensor elements that all have the same number of dimensions, but got tensor elements with dimensions: z, C)orderri  rj  rk  rl  rm  rn  r2   r   ))r   r   rK  r   r   r;   r<   rm   emptyobject_r  rT  	enumerateasarrayr  r   rz  ravelr   _concat_ndarraysr   rs   ru  r   r   rv  rw  rx  ry  r  r}  r~  r   r  insertr   LargeListArrayr   r  r   r  )rS   r[  raveledshapessizesr  iasize_offsets
total_sizer)  r   pa_scalar_typer  offset_arraydata_storage_arrayr  r  type_s                      r*   r   z)ArrowVariableShapedTensorArray.from_numpy  s$   0 #eRZ899 	MAEcM M  
 s88q==TUUU (3s882:666#c(("*555	#c(("(333cNN 	0 	0DAq
1AAFdNN 9(,9 9019 9   6DF1IvE!H !3///GAJJ y''!"%

 'w//!,U338n-- 	7#%%3&&3=E+A+A ( %( (    Yu~66N:!!k&6&:&:
 +kHEEEK l;//X**J{(;
 

 yq!44x--.::<TT hv&& .,,-W
 

 .ndCC(((r)   Tr  c                 h   | j                             d          }| j                             d          }|j        j        |                                d         |                                }|j                                        }t          fdt          ||          D                       S )a
  
        Convert the entire array of tensors into a single ndarray.

        Args:
            zero_copy_only: If True, an exception will be raised if the conversion to a
                NumPy array would require copying the underlying data (e.g. in presence
                of nulls, or for non-primitive types). This argument is currently
                ignored, so zero-copy isn't enforced even if this argument is true.

        Returns:
            A single ndarray representing the entire array of tensors.
        r2   r   r  c                 :    g | ]\  }}t          ||          S r(   )r'  )r   r   r%  data_array_bufferdata_value_types      r*   r   z;ArrowVariableShapedTensorArray.to_numpy.<locals>.<listcomp>	  s<       !E6 #5/6CTUU  r)   )	r  fieldr<   r  r&  	to_pylistr   r   zip)rK   r  r  shapes_arrayr  r   r  r  s         @@r*   r  z'ArrowVariableShapedTensorArray.to_numpy  s     \''//
|))'22$/4&..003''))$..00$    %(%9%9  
 
 	
r)   r  c                    | j         j        k    r| S | j         j        k     rt          d| j          d          t          | j         j                  }| j                            d          }| j                            d          }t          j        fd|	                                D                       }t          j
                            ||g          }|                    |          S )Nr  r  r2   r   c                 0    g | ]}t          |          S r(   )r  )r   r   r  s     r*   r   zMArrowVariableShapedTensorArray.to_var_shaped_tensor_array.<locals>.<listcomp>"  s$    WWW+At44WWWr)   )r<   r  r;   r   r  r  r  rs   r   r  r  r   r  )rK   r  r  r  r  expanded_shapes_arrayr  s    `     r*   r  z9ArrowVariableShapedTensorArray.to_var_shaped_tensor_array  s    49>!!KDIN""SSSDSS   4DI4I4PP \''//
|))'22
 !#WWWWl>T>T>V>VWWW!
 !
 .,,j:O-PQQ%%g...r)   Nr  )r#   r$   r%   r`   rs   r   rT  r  rb   r   r   r   r
   r   r   r   r  r>  r  r(   r)   r*   r]  r]  n  s          !,,o)
D$4eBJ6GGHo)	)o) o) o) [o)b
 
t 
 
 
 
</s /7W / / / / / /r)   r]  r   .r  c                 \    |t          |           k    sJ d|t          |           z
  z  | z   S )N)r"   r   )r   r  s     r*   r  r  *  s6     3u::4#e**$%--r)   r   c                    | s
J d            t          |           dk    r| d         S d | D             }d | D             }t          |          dk    r"t          j                            d|            t          |          dk    rt	          t          |                     S t          |                                t          d |D                                 S )	zYUnifies provided tensor types if compatible.

    Otherwise raises a ``ValueError``.
    z%List of tensor types may not be emptyr"   r   c                     h | ]	}|j         
S r(   )r   r   ts     r*   	<setcomp>z%unify_tensor_types.<locals>.<setcomp>E  s    %%%!ag%%%r)   c                     h | ]	}|j         
S r(   )r  r  s     r*   r  z%unify_tensor_types.<locals>.<setcomp>F  s    111aAM111r)   z6Can't unify tensor types with divergent scalar types: c              3   4   K   | ]}t          |          V  d S r/   r   )r   r   s     r*   	<genexpr>z%unify_tensor_types.<locals>.<genexpr>W  s(      ((AQ((((((r)   )r   r  )	rm   rs   libArrowTypeErrornextiterr   popmax)r   r  scalar_typess      r*   unify_tensor_typesr  7  s     999995
5zzQQx%%u%%%F115111L <1f##LULL
 
 	

 6{{aDKK   (   (((((((	   r)   arrsc                    t                      }t                      }| D ]L}t          |j        |          r|                    |j                   2t          d|j         d| d          t          |          dk    r| S t          |          }t          |t                    sJ g }| D ]0}|	                    |
                    |j                             1|S )Nz)Trying to unify unsupported tensor type: z (supported types: r   r"   r  )r   setr   r<   addr;   rm   r  r   appendr  r  )r  supported_tensor_typesdistinct_types_r[  unified_tensor_typeunified_arrss         r*   unify_tensor_arraysr  [  s    >?? eeO  ch 677 	))))rCHrrYorrr   ?q   -_==)+HIIIIIL 
 
**0C0H*II	
 	
 	
 	
 r)   Farraysr\  ensure_copyc                     | s
J d            t          |           dk    r
|s| d         S t          |           }t          j        d |D                       }|d         j        }|                    |          S )a  
    Concatenates multiple tensor arrays.

    NOTE: If one or more of the tensor arrays are variable-shaped and/or any
    of the tensor arrays have a different shape than the others, a variable-shaped
    tensor array will be returned.

    Args:
        arrays: Tensor arrays to concat
        ensure_copy: Skip copying when ensure_copy is False and there is exactly 1 chunk.

    Returns:
        Either ``ArrowTensorArray`` or ``ArrowVariableShapedTensorArray`` holding
        all of the given tensor arrays concatenated.
    z&List of tensor arrays may not be emptyr"   r   c                     g | ]	}|j         
S r(   )r  )r   cs     r*   r   z(concat_tensor_arrays.<locals>.<listcomp>  s    BBBa	BBBr)   )rm   r  rs   r   r<   r  )r	  r
  unified_arraysr  unified_array_types        r*   concat_tensor_arraysr    s    * ;;;;;6
6{{aay )00NBB>BBBCCG'*/((111r)   c                 $   t          |           dk    s
J d            t          |           dk    r| d         S t          |           st          j        |           S | d         j        }t          | d                   }t          |          }t          | d                   |z
  }|t          d | D                       z   }|                    t          j	                  
                    d          }|||                             |          }|S )a_  Concatenates provided collection of ``np.ndarray``s in either of the following
    ways:

        - If provided ndarrays are contiguous, 1D views sharing the same dtype,
        living w/in the same base view, these will be concatenated zero-copy
        by reusing underlying view

        - Otherwise, ``np.concatenate(arrays)`` will be invoked
    r   z0Provided collection of ndarrays may not be emptyr"   c              3   $   K   | ]}|j         V  d S r/   )nbytes)r   r  s     r*   r  z#_concat_ndarrays.<locals>.<genexpr>  s$      77Q777777r)   r  )rm   _are_contiguous_1d_viewsr   r   r   _get_root_base_get_buffer_addresssumviewr  reshape)r  r   basebase_ptr
start_byteend_byte	byte_viewouts           r*   r  r    s     t99q===L===
4yyA~~Aw%d++ $~d###GME$q'""D"4((H$T!W--8JC77$777777H 		"(##++B//I
Jx'
(
-
-e
4
4CJr)   c                 V   | d         j         }t          | d                   }t          | d                   }| D ]n}|j        dk    s(|j         |k    s|j        j        rt          |          |ur dS |j        dk    rDt          |          }||k    r dS ||j        |j        z  z   }odS )Nr   r"   FT)r   r  _get_base_ptrr  rr  rs  rz  ry  )r  r   r  expected_addrr  buffer_addrs         r*   r  r    s    GME$q'""D!$q'**M > > FaKKw%7'  a  ,,556Q;;#A&&-''55#afu~&==4r)   r  c                      t          |           S r/   )r  )r  s    r*   r!  r!    s    q!!!r)   c                     | }t          |j        t          j                  r&|j        }t          |j        t          j                  &|j        |n|S r/   )r   r  r   r   )r  bs     r*   r  r    sM    	A
QVRZ
(
( F QVRZ
(
( "11)r)   r[  c                 (    | j         d         d         S )zDGet the address of the buffer underlying the provided NumPy ndarray.r2   r   )__array_interface__)r[  s    r*   r  r    s    "6*1--r)   c                    t           j                            |          r|j        }n
|j        dz  }||z  }t           j                            |          r|dz  }|dz  }d|t	          j        |           z   dz
  dz  z   }t	          j        |ft          j        ||          }	t	          j        |	d          }	t	          j        | t          j	        |	|          S |
                                }
t           j                            |          r$t	          j        d|j        t          z             }
t	          j        | |
||          S )Nr  r"   r  rm  rn  r  )rs   r   r|  r  r   r{  r   r  r  r  r  r  r   r  r  )r   r  r%  r)  r  data_offsetr  r  r  r[  r  s              r*   r'  r'    sQ   	x:&& 6 '0 '0A5#f,K	x:&& Q )A-!Ao
 $%rwu~~)E)Ia(O#P j%'(%	
 
 
 mC(333z%rxKPPPP**,,I	x$$Z00 YHW*"7;U"UWWXX	:e9[UUUUr)   )r   )r2   )F)tabcrc   r   r9   r   rw  rF   r   r   enumr   typingr   r   r   r	   r
   r   r   r   numpyr   pyarrowrs   packaging.versionr   parse_versionray.cloudpickler6   ray._private.arrow_utilsr   r   ray._private.ray_constantsr   $ray.air.util.object_extensions.arrowr   r   r   $ray.air.util.tensor_extensions.utilsr   r   r   r    ray.data._internal.numpy_supportr   r   ray.utilr   ray.util.annotationsr   r   ray.util.commonr   r   9MIN_PYARROW_VERSION_CHUNKED_ARRAY_TO_NUMPY_ZERO_COPY_ONLYr0   r  r!   r&   r'   r  re   	getLoggerr#   r   r   r1   ra   rp   r=   ABCrA   r8   ri   r   rt   rw   r   r   r   r   r   TimestampTyper   r?  r   r   r   r   r   r   ExtensionTyper   r   r   ExtensionScalarr  ExtensionArrayr}   r   r]  r>  r  AnyArrowExtTensorTyper  r  r  r  r  r!  r  r  r'  register_extension_typerT  ArrowKeyErrorr(   r)   r*   <module>rE     s   



           



                       P P P P P P P P P P P P P P P P P P P P         4 4 4 4 4 4 % % % % % % P P P P P P P P 2 2 2 2 2 2         
                         8 8 8 8 8 8 8 8 % % % % % %      &%'' =JM(<S<S 91>x1H1H .      4    (<';{NPQRRVWWW 		)( ( $ /:k// / + 
	8	$	$S4 S S S S 5 c     P
 P
 P
 P
 P
cg P
 P
  P
f 	" 	" 	" 	" 	"9 	" 	" 	" 7eDIrx/0017X7 7 7 7    RBcBJ	9:RBILRBXRB RB RB ! RBj4>cBJ./4>>A4>X4> 4> 4> 4>nP$uRXr5N/O*P P P P P*:&(&6EH   4<cBJ./<bk< < < <B 
						





! ! &cBJ./bk   (    . . . , , ,lJ lJ lJ lJ lJ+R-=lJ lJ lJ^ V& & & & &4 & & &< W& & & & &6 & & &4 V< < < < <* < < < VR/ R/ R/ R/ R/r( R/ R/ R/l WRJ RJ RJ RJ RJ+R-=RJ RJ RJj Wx/ x/ x/ x/ x/R%6 x/ x/ x/v.c?."%.
38_. . . . &(EE 
     +,       !  F    !
u%'EEF
G!	% "@@
AB! ! ! ! !H      2  2KLM 2 2 ?@ 2  2  2 !  2F5T"*-=!=> 2:    B5T"*5E)E#F 4    B"RZ "C " " " "
*bj *RZ * * * *.RZ .C . . . .
(V (V (VV	 BtXRXZZ@@AAAB00xrxzzBBCCCB<<XRXZZKKLLLLL	 	 	 	DD	s   A5T 
T"!T"