
    PigF                        d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlZ	d dl
Z
dZ	 d dlmZ d dlmZ n# e$ r dZdZY nw xY w	 d dlmZ d dlmc mZ n# e$ r dZY nw xY we
j        j        ZdZdZd	Zd
ZeeeeiZdZdZ g dZ!eddgiZ"d Z#ee"fdZ$d Z%efdZ&d Z'ed             Z(d(dZ)e
j        *                    ed          d             Z+e
j        *                    ed          e
j        ,                    dddg          d                         Z-e
j        *                    ed          e
j        ,                    dg d          d                         Z.e
j        *                    ed          e
j        ,                    dg d           d!                         Z/e
j        *                    ed          e
j        ,                    dg d"          d#                         Z0e
j        *                    e d$          d%             Z1e
j        *                    ed          d&             Z2e
j        *                    ed          d'             Z3dS ))    N)contextmanager)	timedeltaFInMemoryKmsClientTs   0123456789112345
footer_keys   1234567890123450col_keys   2345678901234501col2_keyyearn_legsanimalr   r   c                  >    t          j        g dg dg dd          S )N)i      r     r   )   r      r      d   )FlamingoParrotDogHorsezBrittle stars	Centipeder
   )patable     y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/pyarrow/tests/test_dataset_encryption.pycreate_sample_tabler    :   sA    8888***  	
 	
  r   c           	      R    t          j        | d|dt          d          d          S )NF
AES_GCM_V1g      @)minutes   )r   plaintext_footercolumn_keysencryption_algorithmcache_lifetimedata_key_length_bits)peEncryptionConfigurationr   )r   r&   s     r   create_encryption_configr,   K   s9    %) ---    r   c                  ,    t          j        d          S )Ni,  )r(   )r*   DecryptionConfigurationr   r   r   create_decryption_configr/   W   s    %S9999r   c                 d    t          j        d |                                 D                       S )Nc                 @    i | ]\  }}||                     d           S )zUTF-8)decode.0key_namekeys      r   
<dictcomp>z0create_kms_connection_config.<locals>.<dictcomp>]   s:     
 
 
# cjj))
 
 
r   )custom_kms_conf)r*   KmsConnectionConfigitems)keyss    r   create_kms_connection_configr<   [   s@    !
 
!%
 
 
   r   c                      t          |           S Nr   )kms_connection_configurations    r   kms_factoryr@   d   s    9:::r   c              #      K   | rd V  d S t          j        ||          5  d V  d d d            d S # 1 swxY w Y   d S )Nmatch)pytestraises)success
error_typerC   s      r   cond_raisesrH   h   s       ]:U333 	 	EEE	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   377c                 R    rVt          di t          t          t          it          di t          t          gi                    d          d         }nt          t          d }	 d fd	} j        }d                                 D             fd|D             }t                    dk    sJ t          |          dk    sJ t          t          i}d                                 D             } |||d            ||d            |||d            |||dd            ||dd            |||dd           t                    d	k    r                                D ]\  }|D ]}}	|	                    d          d         }	fd
                                D             }
|	|k    }||	gz   } ||
|d            ||
||            ||
d            ||
|d           ~ ||d            |d            ||d           d S )N.r   Tc                    }t          t                    }t                      }t          |          }t          |           }t	          j        t                    }	t          j        |	||          }
t          j	        |	||          }t          j                                        }|                    |
          }t          j                    }|                    d           t          j        d|||           t          j                                        }t%          j        t(          d          5  t          j        d||           d d d            n# 1 swxY w Y   t          j        |          }t          j                            |	          }t-          |t.          d
          5  t          j        d||          }t-          |t.          d
          5                      |                              |                    |                    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |	                    ||          }t          j        |          }t          j                            |	          }t-          |t.          d
          5  t          j        d||          }t-          |t.          d
          5                      |                              |                    |                    sJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nencryption_config/sample_datasetdatabase_dirformatfile_options
filesystemzno decryptionrB   rS   rU   decryption_configdefault_fragment_scan_optionszUnknown master key)decryption_properties)r,   FOOTER_KEY_NAMEr/   r<   r*   CryptoFactoryr@   dsParquetEncryptionConfigParquetDecryptionConfigr   datasetParquetFileFormatmake_write_optionsfs_MockFileSystem
create_dirwrite_datasetrD   rE   IOErrorParquetFragmentScanOptionsrH   
ValueErrorselectequalsto_tablefile_decryption_properties)	read_keysread_columnsto_table_successdataset_success
write_keysrM   rX   encrypt_kms_connection_configdecrypt_kms_connection_configcrypto_factoryparquet_encryption_cfgparquet_decryption_cfgpformatwrite_optionsmockfspq_scan_optsra   r[   r&   r;   r   s                     r   assert_decryptsz>do_test_dataset_encryption_decryption.<locals>.assert_decrypts}   s    
4_kRR466(DZ(P(P%(DY(O(O%)+66!#!;9;L"
 "
 "$!;9;L"
 "

 *..00224 3 
 
 #%%#
%&	
 	
 	
 	
 *..00]7*:;;; 	L 	LJ'FKKKK	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 44
 
 
 *..*6 / 
 
 *<PQQQ 	Y 	Yj!1'fUUUG-zAUVVV Y Y||L11889I9I,9W9WXXXXXXY Y Y Y Y Y Y Y Y Y Y Y Y Y Y	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y !/ I I)+<!> !>4"7
 
 
 *..*6 / 
 
 *<PQQQ 	Y 	Yj!1'fUUUG-zAUVVV Y Y||L11889I9I,9W9WXXXXXXY Y Y Y Y Y Y Y Y Y Y Y Y Y Y	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Ys~   8EE #E 3/I">H-!I-H1	1I4H1	5III1/M >L+M+L/	/M2L/	3MM
Mc                 T    g | ]%\  }}|D ]}|                     d           d         &S )rJ   r   )split)r4   r5   column_namescolumn_names       r   
<listcomp>z9do_test_dataset_encryption_decryption.<locals>.<listcomp>   sT     ? ? ?"8(L1=? ?"- *//44Q7 ? ? ? ?r   c                 P    g | ]"}|vr                     | d           |#S )NrJ   )
startswith)r4   r   encrypted_column_namesextra_column_paths     r   r   z9do_test_dataset_encryption_decryption.<locals>.<listcomp>   sY     U U U"-!,4J!J!J08#4#?#?;@Q@Q@Q#R#R  9 *  988r   c                 0    i | ]\  }}|t           k    ||S r   r\   r3   s      r   r7   z9do_test_dataset_encryption_decryption.<locals>.<dictcomp>   s3     8 8 8)Hc#66 !#666r   F   c                 2    i | ]\  }}|t           fv ||S r   r   )r4   r5   r6   column_key_names      r   r7   z9do_test_dataset_encryption_decryption.<locals>.<dictcomp>   s<     P P P!.3 (_o,N N N &s N N Nr   r   )T)dictKEYSEXTRA_COL_KEY_NAMEEXTRA_COL_KEYCOLUMN_KEYSr   r   r:   lenr\   
FOOTER_KEY)r   r   extra_column_namer}   all_column_namesplaintext_column_namesfooter_key_onlycolumn_keys_onlycolumn_key_column_namesencrypted_column_namero   plaintext_and_one_successplaintext_and_oner   r&   r   r;   s   ``           @@@@r   %do_test_dataset_encryption_decryptionr   q   sw    !BBdB1=ABBVV[V-?BSAT,UVV-33C88;!  	DY DY DY DY DY DY DY DYN )? ?<G<M<M<O<O? ? ?U U U U U1AU U U
 %&&****%&&****&
3O8 8-1ZZ\\8 8 8 OO%;TBBBOO%;UCCCOO%5u=== O$&<eUKKKO$&<eUKKKO$&6uEEE ;!8C8I8I8K8K 	D 	D4O4)@ D D% )>(C(CC(H(H(K%P P P P26**,,P P P	 -BEV,V)$:>S=T$T!	+A4HHH	+<>WXXX	+A5III	+;UCCCC%D* OD0$777OD0$777OD*D11111r   z+Parquet Encryption is not currently enabled)reasonc                  <    t          t                                 d S r>   )r   r    r   r   r   "test_dataset_encryption_decryptionr      s     **=*?*?@@@@@r   r   listzlist.list.elementc                     t          j        g dg dg ddgdgdggt          j        t          j                                        }t	                                          d|          }t          ||            d S )	N)r   r      )r   r      )      	   typer   )r   arraylist_int32r    append_columnr   )r   	list_datar   s      r   test_list_encryption_decryptionr     s    
 	IIIyyy2$rd;Xbhjj!!  I  !!//	BBE)%=====r   )mapzmap.key_value.keyzmap.key_value.valuec                    t          j        t          j                    t          j                              }t          j        ddgddgddgdgg g g|          }t                                          d	|          }t          ||            d S )
N)k1r   )k2r   )r   r   )k3r   )r   r   )r   r   )k4r   r   r   )r   map_stringr   r   r    r   r   )r   map_typemap_datar   s       r   test_map_encryption_decryptionr     s     wry{{BHJJ//Hx	"Y	$:Y	<RKR	
   H  !!//x@@E)%=====r   )structz	struct.f1z	struct.f2c                    dt          j                    fdt          j                    fg}t          j        |          }t          j        g d|          }t                                          d|          }t          ||            d S )Nf1f2)r   oner   twor   threer   fourr   fiver   sixr   r   )r   r   r   r   r   r    r   r   )r   struct_fieldsstruct_typestruct_datar   s        r   !test_struct_encryption_decryptionr   (  s    
 BHJJ'$	)<=M)M**K(TTT  K  !!//+FFE)%=====r   )colzcol.list.element.key_value.keyz#col.list.element.key_value.value.f1z#col.list.element.key_value.value.f2c                    dt          j                    fdt          j                    fg}t          j        |          }d}d}d}d}d}d}t          j        t          j                    |          }	||d	}
d
|i}d|i}||d}t          j        |	          }|
|g}|g}|g}t          j        ||d |d d g|          g}t                                          d|          }t          ||            d S )Nr   r   r   r   r   r   r   r   )r   r   r   r   )r   r   r   r   )
r   r   r   r   r   r   r   r    r   r   )r   r   r   struct1struct2struct3struct4struct5struct6r   map1map2map3map4	list_typelist1list2list3r   r   s                       r   &test_deep_nested_encryption_decryptionr   8  s    BHJJ'$	)<=M)M**KGGGGGGwrxzz;//H7##Dw<Dw<D7##D""I4LEFEFE5%udDA	RRRSI!!//yAAE)%=====r   z'Parquet Encryption is currently enabledc                      t           j                                        } t          j        t
                    5  |                     d          }ddd           dS # 1 swxY w Y   dS )zTest write_dataset with ParquetFileFormat and test if an exception is thrown
    if you try to set encryption_config using make_write_optionsz
some valuerL   N)r   ra   rb   rD   rE   NotImplementedErrorrc   )ry   _s     r   -test_write_dataset_parquet_without_encryptionr   _  s     j**,,G	*	+	+ G G&&&FFG G G G G G G G G G G G G G G G G Gs   AA #A c                      G d dt           j                  d} t          j                            t          j        d t          |           D             t          j                              gdg          }t          j                    }t          j	        fd          }t          j
        d	d	dgid
dd          }t          j        |||          }t          j        ||t          j                              }t          j        |          }t          j        |          }|                    |          }	|                    |          }
t'          j                    }|                    d           d}t          j        ||||	|           |dz   }t/          j        ||
|                                          }||k    sJ t          j        |||          }|                                }||k    sJ dS )z9Test encryption and decryption of a large number of rows.c                   6    e Zd ZdededefdZdededefdZdS );test_large_row_encryption_decryption.<locals>.NoOpKmsClient	key_bytesr   returnc                 .    t          j        |          }|S r>   )base64	b64encode)selfr   r   bs       r   wrap_keyzDtest_large_row_encryption_decryption.<locals>.NoOpKmsClient.wrap_keyu  s     ++AHr   wrapped_keyc                 .    t          j        |          }|S r>   )r   	b64decode)r   r   r   r   s       r   
unwrap_keyzFtest_large_row_encryption_decryption.<locals>.NoOpKmsClient.unwrap_keyy  s     --AHr   N)__name__
__module____qualname__bytesstrr   r   r   r   r   NoOpKmsClientr   t  sf        	e 	 	 	 	 	 		% 	C 	E 	 	 	 	 	 	r   r   i  c                 4    g | ]}t          j                     S r   )random)r4   r   s     r   r   z8test_large_row_encryption_decryption.<locals>.<listcomp>  s    777V]__777r   r   foo)namesc                                  S r>   r   )r   r   s    r   <lambda>z6test_large_row_encryption_decryption.<locals>.<lambda>  s     r   UNIMPORTANT_KEYTF   )r   r&   double_wrappingr%   r)   rW   rY   rL   rN   zlarge-row-test-dataset)rS   rT   rU   z/part-0.parquetr[   rU   rV   N)r*   	KmsClientr   Tablefrom_arraysr   rangefloat32r9   r]   r+   r^   r_   r`   r.   ri   rb   rc   rn   rd   re   rf   rg   pqParquetFilereadra   rm   )	row_countr   
kms_configrv   rM   
pqe_config
pqd_configscan_optionsfile_formatrz   rn   r{   path	file_path	new_tablera   r   s                   @r   $test_large_row_encryption_decryptionr  n  sY           IH  	77eI&6&6777

 

 

 	 '	 !  E '))J%&?&?&?&?@@N2$&0    +
$5 J +
B$>$@$@ J 0:NNNL&\RRRK22Z2PPM!/!J!J:!V!V!!F
c#DUD"/FD D D D ((I)C   dff  IjkfEEEG  ""IIr   c                     t                      } t                      }t                      }t                      }t	          j        t                    }t          j        |||          }t          j	        |||          }t          j                                        }|                    |ddg          }t          j                    }	|	                    d           t          j        | d|||	           t          j        |          }
t          j                            |
          }t          j        d||		          }|                                D ]u}|                    |||j        |	          }t-          j        |j        ||	
          5 }t1          |j        j                  D ]}|j                            |          }|                    d          j        J |                    d          j        j        dk    sJ |                    d          j        j        dk    sJ |                    d          j        J |                    d          j        j        dk    sJ |                    d          j        j        dk    sJ |                    d          j        J 	 d d d            n# 1 swxY w Y   wd S )Nr   r   )rM   write_statisticsrN   rO   rP   rW   rY   rV   r  r   r   r   r   r   r   ) r    r,   r/   r<   r*   r]   r@   r^   r_   r`   r   ra   rb   rc   rd   re   rf   rg   ri   get_fragmentsrn   r  r
  r  r  metadatanum_row_groups	row_groupcolumn
statisticsminmax)r   rM   rX   kms_connection_configrv   rw   rx   ry   rz   r{   r|   ra   fragmentr[   parquet_filerg_idxr  s                    r   7test_dataset_encryption_with_selected_column_statisticsr%    s8     !!E0220228::%k22N7-/@   7-/@  j**,,G..0 (+ /  M
 !!F
c!"    00  L j***VVGj)'fMMMG))++ > > . I I!#4hmV!M !M^M"7
 
 
 	>  5 DEE > >(1;;FCC	 ''**5AAA ''**59TAAAA ''**59TAAAA ''**5AAA ''**59Q>>>> ''**59S@@@@ ''**5====>	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	>> >s   D!K  K	K	r>   )4r   
contextlibr   datetimer   r   
pyarrow.fsrd   pyarrowr   rD   encryption_unavailablepyarrow.parquetparquetr
  pyarrow.datasetra   r^   ImportError pyarrow.tests.parquet.encryptionr   pyarrow.parquet.encryption
encryptionr*   mark
pytestmarkr   r\   COL_KEYCOL_KEY_NAMEr   r   r   COLUMNSr   r    r,   r/   r<   r@   rH   r   skipifr   parametrizer   r   r   r   r   r  r%  r   r   r   <module>r9     s  $  % % % % % %                                   	B	BBB"BBBBBB++++++++++ " " "!"
 [ 
 !

\7;# 
&
&
&h12  " )8[ 	 	 	 	: : : '+    ; ; ;   J2 J2 J2 J2Z #P   A A A #P    1D(EFF> > GF > 8    FFF > > 	 > #P    (L(L(LMM	> 	> NM 	> 8       > > 	 >4 'P   	G 	G 	G #P   8 8 8v #P   => => => => =>s   3 	??A AA