
    Pin                        d dl Z d dlmZ d dlZ	 d dlmZ d dlmc m	Z
 d dlmZmZmZmZmZ n# e$ r dZdZ
Y nw xY wdZdZdZdZdZe j        j        e j        j        gZ e j        d	
          d             Z e j        d	
          d             Z e j        d	
          d             Zd Zd Zd Z d Z!d Z"	 d1dZ#d Z$d Z%d Z&d Z'd Z(d Z)d Z*d Z+d Z,d Z-d Z.d  Z/d! Z0e j        1                    d"#          d$             Z2d% Z3e j        4                    d& e j5        ddd'(           e j5        d)dd*(           e j5        dd)d+(           e j5        d)d)d,(          g          d-             Z6d. Z7d/ Z8d0 Z9dS )2    N)	timedelta)InMemoryKmsClientMockVersioningKmsClientverify_file_encryptedread_external_keys_to_dictparse_wrapped_keyzencrypted_table.in_mem.parquets   0123456789112345
footer_keys   1234567890123450col_keymodule)scopec                      t           j                            t          j        g d          t          j        g d          t          j        g d          d          } | S )N)         )abc)xyz)paTablefrom_pydictarray)
data_tables    y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/pyarrow/tests/parquet/test_encryption.pyr   r   2   s^    %%Xiii  Xooo&&Xooo&&' '  J
     c                  N    t          j        t          t          ddgi          } | S )Nr   r   )r	   column_keyspeEncryptionConfigurationFOOTER_KEY_NAMECOL_KEY_NAME)basic_encryption_configs    r   r%   r%   <   s3     8"3*
  
 #"r   c                  P    t          j        t          t          ddgid          } | S )Nr   r   F)r	   r   internal_key_materialr    )external_encryption_configs    r   r(   r(   F   s8    !#!;"3*
 $"% "% "% &%r   c                 b    t          j        |           }d }t          j        |          }||fS )z
    Sets up and returns the KMS connection configuration and crypto factory
    based on provided KMS configuration parameters.
    custom_kms_confc                      t          |           S Nr   kms_connection_configurations    r   kms_factoryz1setup_encryption_environment.<locals>.kms_factoryX        !=>>>r   )r!   KmsConnectionConfigCryptoFactory)r+   kms_connection_configr1   crypto_factorys       r   setup_encryption_environmentr7   Q   sD    
 2?SSS? ? ? %k22N .00r   c                     ||                     d          ||                     d          i}t          |          \  }}	t          | ||||	           ||	fS )zL
    Writes an encrypted parquet file based on the provided parameters.
    UTF-8)decoder7   write_encrypted_parquet)
pathr   footer_key_namecol_key_namer	   r
   encryption_configr+   r5   r6   s
             r   write_encrypted_filer@   a   sz     	**733gnnW--O -I- -)> D*.?1>C C C !.00r   c           	         | t           z  }t          j        t          t          ddgidt          d          d          }|j        du sJ t          ||t          t          t          t          |          \  }}t          |           t          j        t          d          	          }t          ||||          }|                    |          sJ d
S )DWrite an encrypted parquet, verify it's encrypted, and then read it.r   r   
AES_GCM_V1      @minutes   r	   r   encryption_algorithmcache_lifetimedata_key_length_bitsFrJ   N)PARQUET_NAMEr!   r"   r#   r$   r   uniform_encryptionr@   
FOOTER_KEYCOL_KEYr   DecryptionConfigurationread_encrypted_parquetequalstempdirr   r<   r?   r5   r6   decryption_configresult_tables           r   !test_encrypted_parquet_write_readrX   w   s    \!D
 2"3*
 * --- " " " /58888,@j/<W- -)> $ 2 ---/ / /)!6H HL\*******r   c           	         | t           z  }t          j        t          ddt	          d          d          }|j        du sJ t          ||t          t          t          d|          \  }}t          |           t          j
        t	          d                    }t          ||||          }|                    |          sJ d	S )
rB   TrC   rD   rE   rG   )r	   rN   rI   rJ   rK   r   rL   N)rM   r!   r"   r#   r   rN   r@   r$   rO   r   rQ   rR   rS   rT   s           r   )test_uniform_encrypted_parquet_write_readrZ      s    \!D 2") --- " " " /47777,@j/<S- -)> $ 2 ---/ / /)!6H HL\*******r   c                    |j         r|                    ||          }n|                    |||           }|J t          j        | |j        |          5 }|                    |           d d d            d S # 1 swxY w Y   d S )N)encryption_properties)r'   file_encryption_propertiespqParquetWriterschemawrite_table)r<   tabler?   r5   r6   r]   writers          r   r;   r;      s    . <%3%N%N!#4&6 &6"" &4%N%N!#4d&< &<"%111		%,"<
> 
> 
> "AG5!!!" " " " " " " " " " " " " " " " " "s   A99A= A=Tc                 h   |r|                     ||          }n|                     |||           }|J t          j        | |          }|j        dk    sJ t          j        | |          }t          |j                  dk    sJ t          j        | |          }|                    d          S )Ndecryption_propertiesr   Tuse_threads)	file_decryption_propertiesr^   read_metadatanum_columnsread_schemalennamesParquetFileread)	r<   rV   r5   r6   r'   ri   metar`   results	            r   rR   rR      s      <%3%N%N!#4&6 &6"" &4%N%N!#4d&< &<" &111$>@ @ @Dq    ^$>@ @ @Fv|!!!!^$>@ @ @F;;4;(((r   c           	      f   | t           z  }t          j        t          t          ddgidt          d          d          }t          ||t          t          t          t          |           t          |           t          t          t                              d          t          t                              d          i          \  }}t          j        t          d          	          }t          j        t          d
          5  t!          ||||           ddd           dS # 1 swxY w Y   dS )zYWrite an encrypted parquet, verify it's encrypted,
    and then read it using wrong keys.r   r   rC   rD   rE   rG   rH   r9   rL   zIncorrect master key usedmatchN)rM   r!   r"   r#   r$   r   r@   rO   rP   r   r7   r:   rQ   pytestraises
ValueErrorrR   )rU   r   r<   r?   wrong_kms_connection_configwrong_crypto_factoryrV   s          r   +test_encrypted_parquet_write_read_wrong_keyr{      s    \!D
 2"3*
 * --- " " " z?L#W.?A A A $8T00j''00V 9 95!5
 2 ---/ / /	z)E	F	F	F " "#%@ 	" 	" 	"" " " " " " " " " " " " " " " " " "s   D&&D*-D*c                     t          | |           t          j        t          d          5  t	          j        | t          z                                             ddd           dS # 1 swxY w Y   dS )zmWrite an encrypted parquet, verify it's encrypted,
    but then try to read it without decryption properties.no decryptionrt   N)rX   rv   rw   IOErrorr^   ro   rM   rp   rU   r   s     r   0test_encrypted_parquet_read_no_decryption_configr      s     &gz:::	w&6	7	7	7 6 6
w-..335556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   /A((A,/A,c                     t          | |           t          j        t          d          5  t	          j        | t          z             ddd           dS # 1 swxY w Y   dS )zwWrite an encrypted parquet, verify it's encrypted,
    but then try to read its metadata without decryption properties.r}   rt   N)rX   rv   rw   r~   r^   rj   rM   r   s     r   9test_encrypted_parquet_read_metadata_no_decryption_configr     s     &gz:::	w&6	7	7	7 1 1
</0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1   AAAc                     t          | |           t          j        t          d          5  t	          j        | t          z             ddd           dS # 1 swxY w Y   dS )zuWrite an encrypted parquet, verify it's encrypted,
    but then try to read its schema without decryption properties.r}   rt   N)rX   rv   rw   r~   r^   rl   rM   r   s     r   7test_encrypted_parquet_read_schema_no_decryption_configr     s     &gz:::	w&6	7	7	7 / /
w-.../ / / / / / / / / / / / / / / / / /r   c           
          | dz  }t          j        t                    }t          j        t
          d          5  t          ||t          t          t          d|           ddd           dS # 1 swxY w Y   dS )MWrite an encrypted parquet, but give only footer key,
    without column key.z)encrypted_table_no_col_key.in_mem.parquetr	   z4Either column_keys or uniform_encryption must be setrt   r   N)	r!   r"   r#   rv   rw   OSErrorr@   r$   rO   rU   r   r<   r?   s       r   'test_encrypted_parquet_write_no_col_keyr     s     @@D 2"$ $ $ 
w%
& 
& 
& A A 	T:'.?	A 	A 	A	A A A A A A A A A A A A A A A A A As   %A--A14A1c           
      
   | dz  }t          j        t          t          ddgid          }t	          j        t          d          5  t          ||t          t          t          d|           d	d	d	           d	S # 1 swxY w Y   d	S )
r   z=encrypted_table_col_key_and_uniform_encryption.in_mem.parquetr   r   T)r	   r   rN   z2Cannot set both column_keys and uniform_encryptionrt   r   N)	r!   r"   r#   r$   rv   rw   r   r@   rO   r   s       r   ;test_encrypted_parquet_write_col_key_and_uniform_encryptionr   '  s     TTD 2"3*
  ! ! ! 
wR
T 
T 
T A A 	T:'.?	A 	A 	AA A A A A A A A A A A A A A A A A As   %A88A<?A<c                     | dz  }|}t          j                    }d }t          j        |          }t          j        t
          d          5  t          |||||           ddd           dS # 1 swxY w Y   dS )<Write an encrypted parquet, but raise KeyError in KmsClient.(encrypted_table_kms_error.in_mem.parquetc                      t          |           S r-   r.   r/   s    r   r1   z;test_encrypted_parquet_write_kms_error.<locals>.kms_factoryD  s     !!=>>>r   r	   rt   N)r!   r3   r4   rv   rw   KeyErrorr;   rU   r   r%   r<   r?   r5   r1   r6   s           r   &test_encrypted_parquet_write_kms_errorr   ;  s     ??D/ 244? ? ?
 %k22N	x|	4	4	4 G Gj2C 5~	G 	G 	GG G G G G G G G G G G G G G G G G G   A..A25A2c                 ,   | dz  }|}t          j                    } G d dt           j                  fd}t          j        |          }t	          j        t          d          5  t          |||||           ddd           dS # 1 swxY w Y   dS )r   r   c                   $    e Zd ZdZd Zd Zd ZdS )Jtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClientzVA KmsClient implementation that throws exception in
        wrap/unwrap calls
        c                 R    t           j                            |            || _        dS )z%Create an InMemoryKmsClient instance.N)r!   	KmsClient__init__configselfr   s     r   r   zStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.__init__^  s#    L!!$''' DKKKr   c                      t          d          )NCannot Wrap Keyrx   r   	key_bytesmaster_key_identifiers      r   wrap_keyzStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.wrap_keyc  s    .///r   c                      t          d          )NzCannot Unwrap Keyr   r   wrapped_keyr   s      r   
unwrap_keyzUtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.unwrap_keyf  s    0111r   N__name__
__module____qualname____doc__r   r   r    r   r   ThrowingKmsClientr   Y  sK        	 		! 	! 	!
	0 	0 	0	2 	2 	2 	2 	2r   r   c                      |           S r-   r   )r0   r   s    r   r1   zDtest_encrypted_parquet_write_kms_specific_error.<locals>.kms_factoryi  s      !=>>>r   r   rt   N)r!   r3   r   r4   rv   rw   rx   r;   )	rU   r   r%   r<   r?   r5   r1   r6   r   s	           @r   /test_encrypted_parquet_write_kms_specific_errorr   P  s-    ??D/ 2442 2 2 2 2BL 2 2 2 ? ? ? ? ? %k22N	z):	;	;	; G Gj2C 5~	G 	G 	GG G G G G G G G G G G G G G G G G Gs   (B		BBc                     | dz  }|}t          j                    }d }t          j        |          }t          j        t
          d          5  t          |||||           ddd           dS # 1 swxY w Y   dS )z@Write an encrypted parquet, but raise ValueError in kms_factory.0encrypted_table_kms_factory_error.in_mem.parquetc                      t          d          )NCannot create KmsClientr   r/   s    r   r1   zCtest_encrypted_parquet_write_kms_factory_error.<locals>.kms_factory}  s    2333r   r   rt   N)r!   r3   r4   rv   rw   rx   r;   r   s           r   .test_encrypted_parquet_write_kms_factory_errorr   t  s     GGD/ 2444 4 4 %k22N	z6
8 
8 
8 G G 	 j2C 5~	G 	G 	GG G G G G G G G G G G G G G G G G Gr   c                    | dz  }|}t          j                    } G d d          fd}t          j        |          }t          j        t
                    5  t          |||||           ddd           dS # 1 swxY w Y   dS )z_Write an encrypted parquet, but use wrong KMS client type
    that doesn't implement KmsClient.r   c                   $    e Zd ZdZd Zd Zd ZdS )Otest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClientz4This is not an implementation of KmsClient.
        c                     |j         | _        d S r-   )r+   master_keys_mapr   s     r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.__init__  s    #)#9D   r   c                     d S r-   r   r   s      r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.wrap_key      4r   c                     d S r-   r   r   s      r   r   zZtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.unwrap_key  r   r   Nr   r   r   r   WrongTypeKmsClientr     sK        	 		: 	: 	:	 	 		 	 	 	 	r   r   c                      |           S r-   r   )r0   r   s    r   r1   zHtest_encrypted_parquet_write_kms_factory_type_error.<locals>.kms_factory  s    !!">???r   N)r!   r3   r4   rv   rw   	TypeErrorr;   )	rU   r   r%   r<   r?   r5   r1   r6   r   s	           @r   3test_encrypted_parquet_write_kms_factory_type_errorr     s(    GGD/ 244       @ @ @ @ @ %k22N	y	!	! G Gj2C 5~	G 	G 	GG G G G G G G G G G G G G G G G G Gs   A<<B B c            
      l   d } t          j        t          t          ddgidddt	          d          dd	
          } | |           t          j        t                    }t          ddgi|_        d|_        d|_        d|_        t	          d          |_	        d|_
        d	|_         | |           d S )Nc                     t           | j        k    sJ ddg| j        t                   k    sJ d| j        k    sJ | j        sJ | j        rJ t          d          | j        k    sJ | j	        rJ d| j
        k    sJ d S )Nr   r   AES_GCM_CTR_V1      $@rE      )r#   r	   r   r$   rI   plaintext_footerdouble_wrappingr   rJ   r'   rK   )r?   s    r   !validate_encryption_configurationzZtest_encrypted_parquet_encryption_configuration.<locals>.validate_encryption_configuration  s    "3">>>>>Sz.:<HHHHH#4#IIIII 1111$4444&&&*;*JJJJJ$::::'<<<<<<<r   r   r   r   TFr   rE   r   )r	   r   rI   r   r   rJ   r'   rK   r   )r!   r"   r#   r$   r   r   rI   r   r   rJ   r'   rK   )r   r?   encryption_config_1s      r   /test_encrypted_parquet_encryption_configurationr     s    = = = 2"!C:0- ...# 	 	 	 &%&78884"$ $ $'3c3Z&B#/?,+/(*/')24)@)@)@&05-/2,%%&9:::::r   c                     t          j        t          d                    } t          d          | j        k    sJ t          j                    }t          d          |_        t          d          |j        k    sJ d S )Nr   rE   rL   )r!   rQ   r   rJ   )rV   decryption_config_1s     r   /test_encrypted_parquet_decryption_configurationr     s    2 ...0 0 0T"""&7&FFFFF466)24)@)@)@&T"""&9&HHHHHHHr   c                      d } t          j        dddddd          } | |           t          j                    }d|_        d|_        d|_        ddd|_         | |           d S )	Nc                 t    d| j         k    sJ d| j        k    sJ d| j        k    sJ ddd| j        k    sJ d S )N	Instance1URL1MyTokenkey_material_1key_material_2key1key2kms_instance_idkms_instance_urlkey_access_tokenr+   )r5   s    r   validate_kms_connection_configzPtest_encrypted_parquet_kms_configuration.<locals>.validate_kms_connection_config  sq    3CCCCC.?????1BBBBB)3CDD%56 6 6 6 6 6r   r   r   r   r   r   r   r   )r!   r3   r   r   r   r+   )r   r5   kms_connection_config_1s      r   (test_encrypted_parquet_kms_configurationr     s    7 7 7 2#"$$
 
	   #"#8999 466.9+/5,/8,  / /+ #"#:;;;;;r   zNPlaintext footer - reading plaintext column subset reads encrypted columns too)reasonc                 `   | t           z  }t          j        t          t          ddgidd          }t          j        t          t                              d          t          t                              d          i          }d }t          j	        |          }t          |||||           d	S )
zWrite an encrypted parquet, with plaintext footer
    and with single wrapping,
    verify it's encrypted, and then read plaintext columns.r   r   TF)r	   r   r   r   r9   r*   c                      t          |           S r-   r.   r/   s    r   r1   zStest_encrypted_parquet_write_read_plain_footer_single_wrapping.<locals>.kms_factory  r2   r   N)rM   r!   r"   r#   r$   r3   rO   r:   rP   r4   r;   )rU   r   r<   r?   r5   r1   r6   s          r   >test_encrypted_parquet_write_read_plain_footer_single_wrappingr     s     \!D
 2"3*
    2Z..w77'..11
  ? ? ? %k22ND*.?1>C C C C Cr   c           	      "  	 | t           z  }t          ||t          t          t          t
          |          \  }}t          |           t          j                    }t          ||||d          }t          j        j                            |          	t          	                                x}          t          |j        t                             dz   k    sJ t#          	fd|D                       sJ |                    |          sJ dS )zWrite an encrypted parquet file with external key material, verify
    it's encrypted, then read both the table and external store.
    Fr'   r   c                 >    g | ]}                     |          d uS r-   )get_key_material).0kstores     r   
<listcomp>z>test_encrypted_parquet_write_read_external.<locals>.<listcomp>1  s,    GGG!&&q))5GGGr   N)rM   r@   r#   r$   rO   rP   r   r!   rQ   rR   r   _parquet_encryptionFileSystemKeyMaterialStorefor_filerm   get_key_id_setr   allrS   )
rU   r   r(   r<   r5   r6   rV   rW   key_idsr   s
            @r   *test_encrypted_parquet_write_read_externalr     s&   
 \!D,@j/<W"-$ -$)> $244)!6#% % %L "=FFtLLE%..000w11&2<@AAAEG G G GGGGGwGGGHHHHH\*******r   )double_wrap_initialdouble_wrap_rotatedzdouble wrapping)idFzsingle to double wrappedzdouble to singe wrappedzsingle wrappingc                    | t           z  }t          j        t          t          ddgid          }t          j        d          }d }t          j        |          }t          |||||           t          |          |	                    d           |
                    ||	           t          |          t          |           t          |t          j                    ||d
          }	t          v sJ t          v sJ t          v sJ t          v sJ dt          ddffd}
 |
t                      |
t                     |                    |	          sJ dS )a  Tests CryptoFactory.rotate_master_keys

    Note: The CryptoFactory.rotate_master_keys() double_wrapping keword arg
    may be either True (the default) or False regardless of whether
    EncryptionConfig.double_wrapping was set to true (also the default) when
    the external key material store was written. This means double wrapping may
    be set one way initially and then applied or removed during rotation.
    r   r   F)r	   r   r'   r   1)r   c                      t          |           S r-   )r   r/   s    r   r1   z8test_external_key_material_rotation.<locals>.kms_factoryR  s    &'CDDDr   2)r   r   master_key_idreturnNc                     	|          }
r|j         }n|j        }t          |          \  }}}|          }r|j         }n|j        }t          |          \  }}}||k     sJ d S r-   )wrapped_kekwrapped_dekr   )r   before_key_matbefore_key_wrapped_
before_verafter_key_matafter_key_wrapped	after_ver
after_keysbefore_keysr   r   s           r   check_rotated_external_keyszHtest_external_key_material_rotation.<locals>.check_rotated_external_keysr  s    $]3 	<!/!;!/!;,-?@@:q"=1 	: - 9 - 9+,=>>9a I%%%%%%r   )rM   r!   r"   r#   r$   r3   r4   r;   r   refresh_key_access_tokenrotate_master_keysr   rR   rQ   strrS   )reusable_tempdirr   r   r   r<   r?   r5   r1   r6   table_read_after_rotationr
  r  r	  s     ``       @@r   #test_external_key_material_rotationr  5  s   & l*D2"!C:.#+	- - - 2CHHHE E E%k22N   -T22K 223777%%+ & - - -
 ,D11J$ 6
"$$#!% !% !% k))));&&&&j((((:%%%%&3 &4 & & & & & & & & &"  000---67777777r   c           	         | t           z  }t          ||t          t          t          t
          |          \  }}t          |           t          j        t          d                    }t          d          D ]_}|                    ||          }|J t          j        ||          }	|	                    d          }
|                    |
          sJ `dS )	z`Write an encrypted parquet, verify it's encrypted,
    and then read it multithreaded in a loop.rD   rE   rL   2   Nre   Trg   )rM   r@   r#   r$   rO   rP   r   r!   rQ   r   rangeri   r^   ro   rp   rS   )rU   r   r%   r<   r5   r6   rV   iri   rr   rW   s              r   test_encrypted_parquet_loopr    s    \!D
 -Aj/<W-! -!)> $2 ---/ / / 2YY 	/ 	/%3%N%N!#4&6 &6")555(BD D D{{t{44  ......	/ 	/r   c           	         | t           z  }t          ||t          t          t          t
          |          \  }}t          |           t          j        t          d                    }|
                    ||          }~t          j        ||          }|                    d          }	|                    |	          sJ dS )z^
    Test that decryption properties can be used if the crypto factory is no longer alive
    rD   rE   rL   re   Trg   N)rM   r@   r#   r$   rO   rP   r   r!   rQ   r   ri   r^   ro   rp   rS   )
rU   r   r%   r<   r5   r6   rV   ri   rr   rW   s
             r   %test_read_with_deleted_crypto_factoryr    s     \!D,@j/<W-! -!)> $ 2 ---/ / /!/!J!J0"2 "2^$>@ @ @F;;4;00L\*******r   c           	         | t           z  }t          ||t          t          t          t
          |          \  }}t          j        t          d                    }|	                    ||          }t          j        ||          }|                    |          sJ t          j        | |          }|                    |          sJ dS )z>Write an encrypted parquet then read it back using read_table.rD   rE   rL   re   N)rM   r@   r#   r$   rO   rP   r!   rQ   r   ri   r^   
read_tablerS   )	rU   r   r%   r<   r5   r6   rV   ri   rW   s	            r   !test_encrypted_parquet_read_tabler    s    \!D -Aj/<W-! -!)> 2 ---/ / /!/!J!J0"2 "2 ==WXXXL \***** ='AC C CL\*******r   )T):rv   datetimer   pyarrowr   pyarrow.parquetparquetr^   pyarrow.parquet.encryption
encryptionr!    pyarrow.tests.parquet.encryptionr   r   r   r   r   ImportErrorrM   rO   r#   rP   r$   markparquet_encryption
pytestmarkfixturer   r%   r(   r7   r@   rX   rZ   r;   rR   r{   r   r   r   r   r   r   r   r   r   r   r   r   xfailr   r   parametrizeparamr  r  r  r  r   r   r   <module>r*     s  "           E      +++++++++
E E E E E E E E E E E E E E E	    	B	BBB 0 

 K"
K
 h    h# #  # h& &  &1 1 1 1 1 1,+ + +>+ + +6" " "" 26) ) ) ). "  "  "F6 6 61 1 1/ / /A A A"A A A(G G G*!G !G !GHG G G(G G GB ;  ;  ;FI I I< < <:  2 3 3C C3 3CN+ + +2 2T4$5666UD%?@@@T5%>???UE&7888	5:; ;J8 J8; ;J8Z/ / /:+ + +0+ + + + +s   . 	::