o
    'igF                     @   sP  d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlZ	d dl
Z
dZzd dlmZ d dlmZ W n ey?   dZdZY nw zd dlmZ d dlm  mZ W n ey\   dZY nw e
jjZdZdZd	Zd
ZeeeeiZdZdZ g dZ!eddgiZ"dd Z#ee"fddZ$dd Z%efddZ&dd Z'edd Z(d7ddZ)e
jj*eddd d! Z+e
jj*edde
j,d"d#d$gd%d& Z-e
jj*edde
j,d"g d'd(d) Z.e
jj*edde
j,d"g d*d+d, Z/e
jj*edde
j,d"g d-d.d/ Z0e
jj*e d0dd1d2 Z1e
jj*eddd3d4 Z2e
jj*eddd5d6 Z3dS )8    N)contextmanager)	timedeltaFInMemoryKmsClientTs   0123456789112345
footer_keys   1234567890123450col_keys   2345678901234501col2_keyyearn_legsanimalr   r   c                   C   s   t g dg dg ddS )N)i      r     r   )   r      r      d   )FlamingoParrotDogHorsezBrittle stars	Centipeder	   )patable r   r   c/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/pyarrow/tests/test_dataset_encryption.pycreate_sample_table:   s   r   c                 C   s   t j| d|dtddddS )NF
AES_GCM_V1g      @)minutes   )r   plaintext_footercolumn_keysencryption_algorithmcache_lifetimedata_key_length_bits)peEncryptionConfigurationr   )r   r"   r   r   r   create_encryption_configK   s   r(   c                   C   s   t jddS )Ni,  )r$   )r&   DecryptionConfigurationr   r   r   r   create_decryption_configW   s   r*   c                 C   s   t jdd |  D dS )Nc                 S   s   i | ]
\}}|| d qS )zUTF-8)decode.0key_namekeyr   r   r   
<dictcomp>]   s    
z0create_kms_connection_config.<locals>.<dictcomp>)custom_kms_conf)r&   KmsConnectionConfigitems)keysr   r   r   create_kms_connection_config[   s
   r5   c                 C   s   t | S Nr   )kms_connection_configurationr   r   r   kms_factoryd   s   r8   c                 c   sJ    | rd V  d S t j||d d V  W d    d S 1 sw   Y  d S )Nmatch)pytestraises)success
error_typer:   r   r   r   cond_raisesh   s   
"r?   c                    s  r!t di tttit di ttgidd }nttd }	dfdd	}j}dd  D fdd|D }tdksOJ t|dksWJ t	t
i}d	d
  D }|||d ||d |||d |||dd ||dd |||dd tdkr҈ D ]<\ }|D ]5}	|	dd }	 fdd
 D }
|	|k}||	g }||
|d ||
|| ||
d ||
|d qq||d |d ||d d S )N.r   Tc              	      s2  }t t }t }t|}t| }tt}	t|	||}
t	|	||}t
j }|j|
d}t }|d tjd|||d t
j }tjtdd tjd||d W d    n1 sbw   Y  tj|d}t
jj|d	}t|td
d4 tjd||d}t|td
d |||sJ W d    n1 sw   Y  W d    n1 sw   Y  |	||}tj|d}t
jj|d	}t|td
d= tjd||d}t|td
d |||sJ W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nencryption_config/sample_datasetdatabase_dirformatfile_options
filesystemzno decryptionr9   rH   rJ   decryption_configdefault_fragment_scan_optionszUnknown master key)decryption_properties)r(   FOOTER_KEY_NAMEr*   r5   r&   CryptoFactoryr8   dsParquetEncryptionConfigParquetDecryptionConfigr   datasetParquetFileFormatmake_write_optionsfs_MockFileSystem
create_dirwrite_datasetr;   r<   IOErrorParquetFragmentScanOptionsr?   
ValueErrorselectequalsto_tablefile_decryption_properties)	read_keysread_columnsto_table_successdataset_success
write_keysrB   rM   encrypt_kms_connection_configdecrypt_kms_connection_configcrypto_factoryparquet_encryption_cfgparquet_decryption_cfgpformatwrite_optionsmockfspq_scan_optsrV   rP   )r"   r4   r   r   r   assert_decrypts}   sv   




	$z>do_test_dataset_encryption_decryption.<locals>.assert_decryptsc                 S   s(   g | ]\}}|D ]	}| d d qqS )r@   r   )split)r-   r.   column_namescolumn_namer   r   r   
<listcomp>   s    z9do_test_dataset_encryption_decryption.<locals>.<listcomp>c                    s0   g | ]}| vrd u s | ds|qS )Nr@   )
startswith)r-   ru   )encrypted_column_namesextra_column_pathr   r   rv      s    c                 S   s   i | ]\}}|t kr||qS r   rQ   r,   r   r   r   r0      s
    z9do_test_dataset_encryption_decryption.<locals>.<dictcomp>F   c                    s"   i | ]\}}|t  fv r||qS r   rz   r,   )column_key_namer   r   r0      s
    r   )T)dictKEYSEXTRA_COL_KEY_NAMEEXTRA_COL_KEYCOLUMN_KEYSrs   rt   r3   lenrQ   
FOOTER_KEY)r   ry   extra_column_namerr   all_column_namesplaintext_column_namesfooter_key_onlycolumn_keys_onlycolumn_key_column_namesencrypted_column_namerd   plaintext_and_one_successplaintext_and_oner   )r|   r"   rx   ry   r4   r   r   %do_test_dataset_encryption_decryptionq   sZ   G

r   z+Parquet Encryption is not currently enabled)reasonc                   C   s   t t  d S r6   )r   r   r   r   r   r   "test_dataset_encryption_decryption   s   r   ru   listzlist.list.elementc                 C   sR   t jg dg dg ddgdgdggt t  d}t d|}t||  d S )	N)r{   r      )r   r      )      	   typer   )r   arraylist_int32r   append_columnr   )ru   	list_datar   r   r   r   test_list_encryption_decryption  s    r   )mapzmap.key_value.keyzmap.key_value.valuec                 C   sX   t t  t  }t jddgddgddgdgg g g|d}t d	|}t||  d S )
N)k1r{   )k2r   )r   r   )k3r   )r   r   )r   r   )k4r   r   r   )r   map_stringr   r   r   r   r   )ru   map_typemap_datar   r   r   r   test_map_encryption_decryption  s   r   )structz	struct.f1z	struct.f2c                 C   sP   dt  fdt  fg}t |}t jg d|d}t d|}t||  d S )Nf1f2)r{   oner   twor   threer   fourr   fiver   sixr   r   )r   r   r   r   r   r   r   r   )ru   struct_fieldsstruct_typestruct_datar   r   r   r   !test_struct_encryption_decryption(  s   
r   )colzcol.list.element.key_value.keyz#col.list.element.key_value.value.f1z#col.list.element.key_value.value.f2c                 C   s   dt  fdt  fg}t |}d}d}d}d}d}d}t t  |}	||d	}
d
|i}d|i}||d}t |	}|
|g}|g}|g}t j||d |d d g|dg}t d|}t	||  d S )Nr   r   r   r   r   r   r   r   )r{   r   r   r   )r   r   r   r   )
r   r   r   r   r   r   r   r   r   r   )ru   r   r   struct1struct2struct3struct4struct5struct6r   map1map2map3map4	list_typelist1list2list3r   r   r   r   r   &test_deep_nested_encryption_decryption8  s(   



r   z'Parquet Encryption is currently enabledc                  C   sF   t j } tt | jdd}W d   dS 1 sw   Y  dS )zTest write_dataset with ParquetFileFormat and test if an exception is thrown
    if you try to set encryption_config using make_write_optionsz
some valuerA   N)r   rV   rW   r;   r<   NotImplementedErrorrX   )rn   _r   r   r   -test_write_dataset_parquet_without_encryption_  s   
	"r   c                     sB  G dd dt j d} tjjtjdd t| D t dgdgd}t  }t 	 fd	d
}t j
dddgidddd}t|||}t||t  }tj|d}tj|d}|j|d}	||}
t }|d d}tj||||	|d |d }tj||
|d }||ksJ tj|||d}| }||ksJ dS )z9Test encryption and decryption of a large number of rows.c                   @   s8   e Zd ZdededefddZdededefddZd	S )
z;test_large_row_encryption_decryption.<locals>.NoOpKmsClient	key_bytesr   returnc                 S      t |}|S r6   )base64	b64encode)selfr   r   br   r   r   wrap_keyu     
zDtest_large_row_encryption_decryption.<locals>.NoOpKmsClient.wrap_keywrapped_keyc                 S   r   r6   )r   	b64decode)r   r   r   r   r   r   r   
unwrap_keyy  r   zFtest_large_row_encryption_decryption.<locals>.NoOpKmsClient.unwrap_keyN)__name__
__module____qualname__bytesstrr   r   r   r   r   r   NoOpKmsClientt  s    r   i  c                 S   s   g | ]}t   qS r   )random)r-   r   r   r   r   rv     s    z8test_large_row_encryption_decryption.<locals>.<listcomp>r   foo)namesc                    s     S r6   r   )r   r   r   r   <lambda>  s    z6test_large_row_encryption_decryption.<locals>.<lambda>UNIMPORTANT_KEYTF   )r   r"   double_wrappingr!   r%   rL   rN   rA   rC   zlarge-row-test-dataset)rH   rI   rJ   z/part-0.parquetrP   rJ   rK   N)r&   	KmsClientr   Tablefrom_arraysr   rangefloat32r2   rR   r'   rS   rT   rU   r)   r^   rW   rX   rc   rY   rZ   r[   r\   pqParquetFilereadrV   rb   )	row_countr   
kms_configrk   rB   
pqe_config
pqd_configscan_optionsfile_formatro   rc   rp   path	file_path	new_tablerV   r   r   r   $test_large_row_encryption_decryptionn  sZ   	



r   c               	   C   s  t  } t }t }t }tt}t|||}t	|||}t
j }|j|ddgd}t }	|	d tj| d|||	d tj|d}
t
jj|
d}tjd||	d	}| D ]}}||||j|	}tj|j||	d
a}t|jjD ]R}|j|}|djd usJ |djjdksJ |djjdksJ |djd usJ |djjdksJ |djjdksJ |djd u sJ qtW d    n1 sw   Y  qYd S )Nr
   r   )rB   write_statisticsrC   rD   rE   rL   rN   rK   r   r   r   r   r{   r   r   ) r   r(   r*   r5   r&   rR   r8   rS   rT   rU   r   rV   rW   rX   rY   rZ   r[   r\   r^   get_fragmentsrc   r   r   r   r   metadatanum_row_groups	row_groupcolumn
statisticsminmax)r   rB   rM   kms_connection_configrk   rl   rm   rn   ro   rp   rq   rV   fragmentrP   parquet_filerg_idxr  r   r   r   7test_dataset_encryption_with_selected_column_statistics  sj   




r  r6   )4r   
contextlibr   datetimer   r   
pyarrow.fsrY   pyarrowr   r;   encryption_unavailablepyarrow.parquetparquetr   pyarrow.datasetrV   rS   ImportError pyarrow.tests.parquet.encryptionr   pyarrow.parquet.encryption
encryptionr&   mark
pytestmarkr   rQ   COL_KEYCOL_KEY_NAMEr~   r   r   COLUMNSr   r   r(   r*   r5   r8   r?   r   skipifr   parametrizer   r   r   r   r   r   r  r   r   r   r   <module>   s   	

 

	

;