
    Pi'G                        d dl Z d dlZd dlmZ 	 d dlmZ d dlmZm	Z	m
Z
mZ n# e$ r dZY nw xY w	 d dlZd dlmZ n# e$ r dxZZY nw xY we j        j        Ze j        j        d             Zd Zd Ze j        j        d             Ze j        j        d             Ze j        j        e j                            d	d ej                    g          d
                         Ze j        j        e j        j        d                         Ze j        j        e j                            d	d ej                    g          d                         Ze j        j        e j        j        d                         Z e j        j        e j        j        d                         Z!e j        j        e j        j        d                         Z"e j        j#        d             Z$d Z%d Z&d Z'd Z(e j                            dej)        dfej)        dfej*        dfej*        dfg          e j                            dddg          d                         Z+e j                            dg d          d              Z,d! Z-dS )"    N)fs)_read_table_test_dataframe_test_table_range_integersc                    t          d          }d|d<   t          j                            |d          }t          j                    }t          j        ||j        d          }g }t          d          D ]d}||d<   t          j                            |d          }|	                    |           |
                    |                                           e|                                 |                                }t          t          j        |                    }t!          j        |d	
          }	t%          j        |                                |	           d S Nd   r   	unique_idFpreserve_index2.6version
   Tignore_index)r   paTablefrom_pandasBufferOutputStreampqParquetWriterschemarangewrite_tableappendcopyclosegetvaluer   BufferReaderpdconcattmassert_frame_equal	to_pandas
tempdirdfarrow_tableoutwriterframesibufresultexpecteds
             }/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/pyarrow/tests/parquet/test_parquet_writer.py#test_parquet_incremental_file_buildr3   ,   s2   			BB{O(&&r%&@@K


!
!Cc;#5uEEEFF2YY ! !;h**2e*DD;'''bggii    
LLNNN
,,..C--..Fyd333H&**,,h77777    c                 X   t          j        dt          j                              t          j        dt          j                              g}t          j        |          }t          j        dg          t          j        dg          g}t           j                            |ddg          }| dz  }t          j	        ||ddd	          5 }t          j        t                    5  |                    |           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )
NPOSdesc   blazsimple_validate_schema.parquetr   snappyspark)r   compressionflavor)r   fielduint32stringr   arrayr   from_arraysr   r   pytestraises
ValueErrorr   )r(   simple_fieldssimple_schemasimple_from_arraysimple_tablepathws          r2    test_validate_schema_write_tablerL   G   s    		$$
%%M
 Im,,M 1#%(9(9:8''(9E6?KKL55D		$"'&.w
@ 
@ 
@ (CD]:&& 	( 	(MM,'''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (s6   D%D;DD	DD	DD#&D#c                    t          j        t                    5  t          j        t          j        dt          j                              g          }t          j        d |           d d d            n# 1 swxY w Y   t          j        t                    5  t          j        | dz  d            d d d            d S # 1 swxY w Y   d S )Nx	some_path)	rC   rD   	TypeErrorr   r   r>   int32r   r   )r(   some_schemas     r2   test_parquet_invalid_writerrS   ]   s,   	y	!	! , ,i#rxzz!:!: ;<<
{+++, , , , , , , , , , , , , , , 
y	!	! 6 6
;.5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s$   AA55A9<A9B??CCc                    t          d          }d|d<   t          j                            |d          }t          j                    }t          j        ||j        d          5 }g }t          d          D ]d}||d<   t          j                            |d          }|	                    |           |
                    |                                           e	 d d d            n# 1 swxY w Y   |                                }t          t          j        |                    }t          j        |d	
          }	t#          j        |                                |	           d S r	   )r   r   r   r   r   r   r   r   r   r   r   r   r    r   r!   r"   r#   r$   r%   r&   r'   s
             r2   test_parquet_writer_context_objrU   g   s   			BB{O(&&r%&@@K


!
!C		#{15	A	A	A %Vr 	% 	%AB{O(..r%.HHK{+++MM"''))$$$$	%% % % % % % % % % % % % % % % ,,..C--..Fyd333H&**,,h77777s   %A7C))C-0C-c                 Z   t          d          }d|d<   t          j                            |d          }t          j                    }d}	 t          j        ||j        d          5 }g }t          d	          D ]y}||d<   t          j                            |d          }|	                    |           |
                    |                                           |d
k    rt          |          z	 d d d            n# 1 swxY w Y   n,# t          $ r}t          |          |k    sJ Y d }~nd }~ww xY w|                                }	t!          t          j        |	                    }
t%          j        |d          }t)          j        |
                                |           d S )Nr
   r   r   Fr   zArtificial Errorr   r   r      Tr   )r   r   r   r   r   r   r   r   r   r   r   r   rE   	Exceptionstrr    r   r!   r"   r#   r$   r%   r&   )r(   r)   r*   r+   
error_textr,   r-   r.   er/   r0   r1   s               r2   .test_parquet_writer_context_obj_with_exceptionr\      s   			BB{O(&&r%&@@K


!
!C#J$c)0&+- - - 	106F2YY 1 1"#; h222e2LL"";///bggii(((66$Z000 1	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1  $ $ $1vv#########$ ,,..C--..Fyd333H&**,,h77777s=   D (BD5D DD D	D 
D6D11D6
filesystemc                    t          d          }t          j                            |d          }t          j                            |d          }t          | dz            }t          | dz            }t          j        ||j        |d          5 }|	                    |           d d d            n# 1 swxY w Y   t          |                                          }t          j        ||           t          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |                                          }t          j        ||           t          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |                                          }t          j        ||           t          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |                                          }t          j        ||           d S )Nr
   Fr   zdata_table.parquetzdata_batch.parquetr   r]   r   )r   r   r   r   RecordBatchrY   r   r   r   r   r   r&   r$   r%   write_batchwrite)	r(   r]   r)   tablebatch
path_table
path_batchr,   r0   s	            r2   "test_parquet_writer_write_wrappersrg      sM    
		BH  E ::EN&&r%&@@EW3344JW3344J		ELZ
 
 
 "	5!!!" " " " " " " " " " " " " " "
 $$..00F&"%%%		ELZ
 
 
 "	5!!!" " " " " " " " " " " " " " "
 $$..00F&"%%%		ELZ
 
 
 	U              
 $$..00F&"%%%		ELZ
 
 
 	U              
 $$..00F&"%%%%%sH   B55B9<B9D55D9<D9F55F9<F9H55H9<H9c                     dd fd} |dz  dz
  d            |dz  d            |dz  dz   d            |dz   dz   d            |dz  dz  d            |d d            |dz   d d           d S )Ni   i   c                 d   t           j                            t          | d          gdg          }|t	          j        |dz             nt	          j        |dz  |           t	          j        dz            }|
n|}|j        |k    sJ t          |	          }t          |dz
            D ]"}|
                    |          j        |k    sJ #| ||dz
  z  z
  }|dk    r%|
                    |dz
            j        |k    sJ d S |
                    |dz
            j        |k    sJ d S )NbrN   )namesztest.parquet)row_group_sizer8   r   )r   r   rB   r   r   r   read_metadatanum_row_groupsminr   	row_groupnum_rows)	data_size
chunk_sizeexpect_num_chunksrc   metadataexpected_chunk_sizelatched_chunk_size	chunk_idx	remainderabs_max_chunk_sizedefault_chunk_sizer(   s            r2   check_chunk_sizez8test_parquet_writer_chunk_size.<locals>.check_chunk_size   s   $$Is++&
 %   N5'N":;;;;N5'N"::VVVV#Gn$<==4>4F00J&*;;;;; !46HII01455 	P 	PI%%i009=OOOOOO!48IA8M!NO	>>%%!A%' ''/3EF F F F F F %%&7!&;<<ERRRRRRr4      r
      r8    )r(   r|   rz   r{   s   ` @@r2   test_parquet_writer_chunk_sizer      s    %)S S S S S S S. '!+-?#-EqIII'!+-?CCC'!+-?#-EqIII'#-/AC/GKKK '!+-?!-CQGGG 'q111'!+T155555r4   c                    t          d          }t          j                            |d          }t	          | dz            }t          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |          
                                }t          j        ||           d S )Nr
   Fr   zdata.parquetr   r_   )r   r   r   r   rY   r   r   r   r   r   r&   r$   r%   )r(   r]   r)   rc   rJ   r,   r0   s          r2   $test_parquet_writer_filesystem_localr      s     
		BH  E ::Ew'((D		elz5
 
 
 "	5!!!" " " " " " " " " " " " " " "
 ((**F&"%%%%%s    BB	Bc                 r   t          d          }t          j                            |d          }| \  }}}t	          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |          	                                }t          j        ||           d S )Nr
   Fr   r   r_   r   r   r   r   r   r   r   r   r   r&   r$   r%   s3_example_fsr)   rc   r   urirJ   r,   r0   s           r2   !test_parquet_writer_filesystem_s3r     s     
		BH  E ::E!MBT		elr5
 
 
 "	5!!!" " " " " " " " " " " " " " "
 ''))F&"%%%%%s   A66A:=A:c                 t   t          d          }t          j                            |d          }| \  }}}t	          j        ||j        d          5 }|                    |           d d d            n# 1 swxY w Y   t          ||          	                                }t          j        ||           d S )Nr
   Fr   r   r   r]   r   r   s           r2   %test_parquet_writer_filesystem_s3_urir     s     
		BH  E ::E!MBT		#u|U	;	;	; "v5!!!" " " " " " " " " " " " " " " "---7799F&"%%%%%s   A55A9<A9c                 ~   t          d          }t          j                            |d          }| \  }}|dz   }t	          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          ||          	                                }t          j        ||           d S )Nr
   Fr   z/test.parquetr   r_   r   r   )s3_example_s3fsr)   rc   r   	directoryrJ   r,   r0   s           r2   #test_parquet_writer_filesystem_s3fsr   +  s    
		BH  E ::E#MB	&D		elr5
 
 
 "	5!!!" " " " " " " " " " " " " " "
 "---7799F&"%%%%%s   A::A>A>c                     t          d          } t          j                    }t          j        t
          d          5  t          j        t          j	                    | j
        |           d d d            d S # 1 swxY w Y   d S )Nr
   zspecified path is file-likematchr   )r   r   LocalFileSystemrC   rD   rE   r   r   r   r   r   )rc   r]   s     r2   ,test_parquet_writer_filesystem_buffer_raisesr   =  s    E#%%J 
z)F	G	G	G 
 

!##U\j	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   .A99A= A=c                     t          j        dg di          }| dz  }t          j        ||j                  5 }|                    |           d d d            n# 1 swxY w Y   t          j        |          }d|j        v sJ |j        d         sJ | dz  }t          j        ||j        d          5 }|                    |           d d d            n# 1 swxY w Y   t          j        |          }|j        J d S )Na)r8   r}   r~   ztest_with_schema.parquets   ARROW:schemaztest_without_schema.parquetF)store_schema)r   rc   r   r   r   r   rm   ru   )r(   rc   path1r,   metapath2s         r2    test_parquet_writer_store_schemar   I  s   Hc999%&&E 00E		%	.	. "&5!!!" " " " " " " " " " " " " " " E""Ddm++++=)))) 33E		%E	B	B	B "f5!!!" " " " " " " " " " " " " " " E""D=     s#   AA!A4CCCc                    t           j                            t          j        g d          gdg          }| dz  }t	          j        ||j                  5 }|                    |           |                    ddd           |                    dd	d
           d d d            n# 1 swxY w Y   t	          j	        |          }|j
        j
        }|d         dk    sJ |d         dk    sJ |d         dk    sJ d S )NrQ   typef0zmetadata.parquet1rN   )key1key223)r   key3s   key1   1s   key2   2s   key3   3)r   r   rB   rA   r   r   r   r   add_key_value_metadataParquetFileru   )r(   rc   rJ   r,   readerru   s         r2   -test_parquet_writer_append_key_value_metadatar   ^  s^   H  "(2G"<"<"<!=vFFE''D		$	-	- B5!!!%%sC&@&@AAA%%sC&@&@AAAB B B B B B B B B B B B B B B ^D!!F'HG$$$$G$$$$G$$$$$$s   AB((B,/B,c           	         t          j        dt          d          i          }t          j        || dz  dd           t          j        || dz  ddd	           t          j        || d
z  ddddd	           t          j        | dz            }t          j        | dz            }t          j        | d
z            }|                    |          sJ |                    |          sJ t          j        | dz            }t          j        | dz            }t          j        | d
z            }|j        |j        k    sJ |j        |j        k    sJ t          |j                  D ]}|	                    |          }	|	                    |          }
|	                    |          }|	j
        |
j
        k    sJ |	j
        |j
        k    sJ |	j        |
j        k     sJ |	j        |j        k     sJ |
j        |j        k     sJ d S )Nr   i zunchunked.parquetFPLAIN)use_dictionarycolumn_encodingzchunked-default.parquetT)r   r   use_content_defined_chunkingzchunked-custom.parquet      min_chunk_sizemax_chunk_size)r   rc   r   r   r   
read_tableequalsrm   rn   rp   rq   total_byte_size)r(   rc   	unchunkedchunked_defaultchunked_customunchunked_metadatachunked_default_metadatachunked_custom_metadatar.   rg_unchunkedrg_chunked_defaultrg_chunked_customs               r2   %test_parquet_content_defined_chunkingr   m  s   Hc5>>*++E N5'$77"'#*, , , , N5'$=="'#*046 6 6 6 N5'$<<"'#*CICI1K 1KL L L L g(;;<<ImG.G$GHHO]7-E#EFFNO,,,,,N+++++ )'4G*GHH!/:S0STT .w9Q/QRR,0H0WWWWW,0G0VVVVV%455 
V 
V)33A665??BB3==a@@$(:(CCCCC$(9(BBBBB +.@.PPPPP+.?.OOOOO!14E4UUUUUU
V 
Vr4   c                 `   t          j        dt          d          i          }| dz  }d}t          j        t
          |          5  ddd}t          j        |||	           d d d            n# 1 swxY w Y   dd
ddfddidfddidfg}|D ]O\  }}t          j        t          |          5  t          j        |||	           d d d            n# 1 swxY w Y   Pt          j        ||d
	           ddd}t          j        |||	           dddd}t          j        |||	           d S )Nr   r
   zchunked-invalid.parquetz2max_chunk_size must be greater than min_chunk_sizer   r   r   r   )r   T)r   unknown_optionzEUnknown options in 'use_content_defined_chunking': {'unknown_option'}r   zEMissing options in 'use_content_defined_chunking': {'max_chunk_size'}r   zEMissing options in 'use_content_defined_chunking': {'min_chunk_size'}r8   )r   r   
norm_level)	r   rc   r   rC   rD   rX   r   r   rE   )r(   rc   rJ   msgcdc_optionscasess         r2   0test_parquet_content_defined_chunking_parametersr     s&   Hc5::&''E..D ?C	y	,	,	, N N)/6JJ
udMMMMN N N N N N N N N N N N N N N  )DAAS	

 y)S	

 y)S	
E " R RS]:S111 	R 	RN5$[QQQQ	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R N5$TBBBB &,vFFKN5$[IIII &,vUVWWKN5$[IIIIIIs$   A//A36A3.CC	C	ztime_type, time_unitsmsusnsutc_flag_valFTc           	         | dz  }g d}t          j        dt          j        | ||                    i          }t          j        d ||          fg          }t	          j        |||          5 }|                    |           d d d            n# 1 swxY w Y   t	          j        ||          }	|	                    d           |		                    |          sJ d S )	Nztime_adjusted_to_utc.parquet)r   {   '  iQ time_colr   )wherer   write_time_adjusted_to_utcr   Tfull)
r   rc   rA   r   r   r   r   r   validater   )
r(   r   	time_type	time_unitfilenametime_valuesrc   r   r,   r0   s
             r2   ,test_arrow_writer_props_time_adjusted_to_utcr     sL    77H***KHBH[yy/C/CDDD  E Y	YYy))*  F 
	#/
 
 
 " 
5!!!" " " " " " " " " " " " " " " ]8F333F
OOO==s   4BBBmax_rows_per_page)r8   r   r
     Nc                 `   | dz  }t          j        t          j        g dt          j                              t          j        g dt          j                              d          }t          j        dt          j                    fdt          j                    fg          }t          j        |||          5 }|                    |           d d d            n# 1 swxY w Y   t          j	        ||	          }|
                    d
           |                    |          sJ d S )Nzmax_rows_per_page.parquet)r8   r}   r~      rW         r   )g      &@g      (@g      *@g      ,@g      .@g      0@g      1@)rN   yrN   r   r   r   r   r   Tr   )r   rc   rA   int8float16r   r   r   r   r   r   r   )r(   r   r   rc   r   r,   r0   s          r2   #test_writer_props_max_rows_per_pager     sp    44HHX+++"'))<<<X@@@rz||TTT   E
 Y	bgii	bjll  F
 
	+
 
 
 " 
5!!!" " " " " " " " " " " " " " " ]8F333F
OOO==s   >C  C$'C$c                    t          j        dt          j        t          d                    i          }t	          j                    }g }dD ]{}|  d| d}t          j        ||j        |          5 }|	                    |           d d d            n# 1 swxY w Y   |
                    |                    |                     ||d         j        |d         j        k    sJ d S )	NrN   i@B )r   r   z/max_rows_per_page_z.parquetr   r   r8   )r   rc   rA   r   r   r   r   r   r   r   r   get_file_infosize)r(   rc   local
file_infosmax_rowsrJ   r,   s          r2   -test_writer_props_max_rows_per_page_file_sizer     sC   HRXeI&&''  E   EJ# 
5 
5@@h@@@<&
 
 
 	& u%%%	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	%--d334444 a=
1 2222222s   3BB	B	).rC   pyarrowr   r   pyarrow.parquetparquetr   pyarrow.tests.parquet.commonr   r   r   r   ImportErrorpandasr"   pandas.testingtestingr$   mark
pytestmarkr3   rL   rS   rU   r\   parametrizer   rg   large_memoryr   r   s3r   r   r   numpyr   r   r   r   r   time32time64r   r   r   r   r4   r2   <module>r     s@  $                 L L L L L L L L L L L L L   	BBB   NB [ 
 8 8 84( ( (,6 6 6 8 8 80 8 8 8> B(  %& %&	  
%&P &6 &6  &6R B(  & &	  
& & &  & 
& 
&  
& & &  &  
 
 
! ! !*% % %+V +V +V\&J &J &JR /YYYY	2   %77    87  B     	  83 3 3 3 3s   # --
< 	AA