
    Pi6                       d dl m Z mZmZ d dlZd dlZd dlZd dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZmZmZmZ d dlmZmZmZmZmZmZmZmZmZm Z m!Z! d dl"m#Z# ej$        %                    ej$        &                    e'                    Z( G d d	e          Z)ej*        d
             Z+ej*        d             Z,ej*        d             Z-ej*        d             Z.ej*        d             Z/ej*        d             Z0ej*        d             Z1ej*        d             Z2ej*        d             Z3dZ4ej*        d             Z5ej*        d             Z6ej*        d             Z7ej*        d             Z8ej*        d             Z9 ej*         ej:        dd           ej:        dd           ej:        dd           ej:        d d!ej;        j<        "           ej:        d#d$ej;        j=        "           ej:        d%d&ej;        j>        "           ej:        d'd(ej;        j6        "           ej:        d)d*           ej:        d+d,           ej:        d-d.           ej:        d/d0           ej:        d1d2           ej:        d3d4ej;        j<        "          g5          d6             Z?ej*        d7             Z@ej*        d8             ZAej*        d9             ZBej*        d:             ZCd; ZDd< ZEd= ZFd> ZGd? ZHej;        j<        d@             ZIdA ZJdB ZKdC ZLdD ZMdE ZNdF ZOdG ZPdH ZQdI ZRdJ ZSdK ZTdL ZUdM ZVdN ZWdO ZXdP ZYdQ ZZdR Z[dS Z\dT Z]dU Z^dV Z_dW Z`ej;        j        ej;        a                    dXdde`fddYe`fdZdejb        fdZd[ejb        fg          d\                         Zcd] Zdd^ Zeej;        j        ej;        a                    d_dde`fddYe`fdZdejf        fdZd[ejf        fg          d`                         Zgej;        j        ej;        a                    dadde`e`fddYe`e`fdZdejb        ejf        fdZd[ejb        ejf        fg          db                         Zhdc Zidd Zjde Zkdf Zldg Zmej;        j=        dh             Znej;        j<        di             Zoej;        j<        dj             Zpej;        j<        dk             Zqej;        j>        dl             Zrej;        j6        dm             Zsej;        a                    dndoedpfdqedrfdsedrfdtedrfduedvfdwedxfdyedxfdzed{fd|ed{fd}ed~fdxedxfd{ed{fdedfg          d             Ztd Zuej;        a                    ddpdxdrd{e'g          d             Zvej;        j<        d             Zwej;        j=        d             Zxd Zyd Zzd Z{d Z|d Z}d Z~d Zd Zd Zd Zd Zej;        j<        d             Zej;        j<        d             Zej;        j<        d             Zej;        j<        d             Zd Zej;        j<        d             Zej;        j<        d             Zej;        j        ej;        j        ej;        j<        d                                     Zej;        j<        ej;                            d          ej;                             e            d          d                                     Zd Zd Zd ZdS )    )datetimetimezone	timedeltaN)urlopen)assert_file_not_found)_filesystem_uriProxyHandler_configure_s3_limited_userrunning_on_musllinux)FileTypeFileInfoFileSelector
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystemFSSpecHandler
copy_files)find_free_portc                   |    e Zd ZddZd Zd Zd Zd Zd Zd Z	d	 Z
d
 Zd Zd Zd Zd Zd Zd Zd Zd Zd ZdS )DummyHandler*   c                     || _         d S N)_value)selfvalues     i/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/pyarrow/tests/test_fs.py__init__zDummyHandler.__init__/   s        c                 Z    t          |t                    r| j        |j        k    S t          S r   
isinstancer   r   NotImplementedr   others     r    __eq__zDummyHandler.__eq__2   *    e.// 	/;%,..r"   c                 Z    t          |t                    r| j        |j        k    S t          S r   r$   r'   s     r    __ne__zDummyHandler.__ne__7   r*   r"   c                     dS )Ndummy r   s    r    get_type_namezDummyHandler.get_type_name<   s    wr"   c                     |S r   r/   r   paths     r    normalize_pathzDummyHandler.normalize_path?   s    r"   c                    g }|D ]}d|v r.|                     t          |t          j                             4d|v r.|                     t          |t          j                             fd|v r.|                     t          |t          j                             d|v r"|                     t                                 t          |S )Nfiledirnotfoundbadtype)appendr   r   File	DirectoryNotFoundobjectIOError)r   pathsinfor4   s       r    get_file_infozDummyHandler.get_file_infoB   s     	 	D~~HT8=99::::$HT8+=>>????t##HT8+<==>>>>d""FHH%%%%r"   c                    |j         dk    r|j        rg S t          |j                   t          dt          j        d          t          dt          j                  g}|j        r |t          dt          j        d          gz  }|S )Nsomedirsomedir/file1{   )sizesomedir/subdir1somedir/subdir1/file2i  )base_dirallow_not_foundFileNotFoundErrorr   r   r<   r=   	recursive)r   selectorinfoss      r    get_file_info_selectorz#DummyHandler.get_file_info_selectorR   s    	))' ;	'(9:::_hm#>>>&(:;;
  	0(-cJJJ E r"   c                 H    |dk    r|du sJ d S |dk    r|du sJ d S t           )NrN   Tnon-recursiveF)r@   )r   r4   rN   s      r    
create_dirzDummyHandler.create_dirb   sK    ;$$$$$$_$$%%%%%%Mr"   c                     |dk    sJ d S )N
delete_dirr/   r3   s     r    rV   zDummyHandler.delete_dirj   s    |######r"   c                 N    |                     d          st          |dk    sJ d S )N/delete_dir_contents)strip
ValueError)r   r4   missing_dir_oks      r    rY   z DummyHandler.delete_dir_contentsm   s2    zz# 	,,,,,,,r"   c                     d S r   r/   r0   s    r    delete_root_dir_contentsz%DummyHandler.delete_root_dir_contentsr   s    r"   c                     |dk    sJ d S )Ndelete_filer/   r3   s     r    r`   zDummyHandler.delete_fileu   s    }$$$$$$r"   c                 &    |dk    sJ |dk    sJ d S )N	move_frommove_tor/   r   srcdests      r    movezDummyHandler.movex   s*    k!!!!y      r"   c                 &    |dk    sJ |dk    sJ d S )Ncopy_file_fromcopy_file_tor/   rd   s      r    	copy_filezDummyHandler.copy_file|   s+    &&&&&~%%%%%%r"   c                     d|v rt          |          | d                    d          }t          j        |          S )Nr9   z:input_streamutf8rM   encodepaBufferReaderr   r4   datas      r    open_input_streamzDummyHandler.open_input_stream   sG    #D)))%%%,,V44t$$$r"   c                     d|v rt          |          | d                    d          }t          j        |          S )Nr9   z:input_filerm   rn   rr   s      r    open_input_filezDummyHandler.open_input_file   sG    #D)))###**622t$$$r"   c                 N    d|v rt          |          t          j                    S Nr9   rM   rp   BufferOutputStreamr   r4   metadatas      r    open_output_streamzDummyHandler.open_output_stream   *    #D)))$&&&r"   c                 N    d|v rt          |          t          j                    S rx   ry   r{   s      r    open_append_streamzDummyHandler.open_append_stream   r~   r"   N)r   )__name__
__module____qualname__r!   r)   r,   r1   r5   rC   rQ   rT   rV   rY   r^   r`   rg   rk   rt   rv   r}   r   r/   r"   r    r   r   .   s'            
  
            $ $ $- - -
  % % %! ! !& & &% % %% % %' ' '
' ' ' ' 'r"   r   c                 H    t          t                      fddd          S )Nc                 2    | z                                   S r   as_posixptempdirs    r    <lambda>zlocalfs.<locals>.<lambda>       'A+//11 r"   Tfspathfnallow_move_dirallow_append_to_filedictr   requestr   s    `r    localfsr      s5    1111!	   r"   c                 |    t          t          t          t                                          fddd          S )Nc                 2    | z                                   S r   r   r   s    r    r   zpy_localfs.<locals>.<lambda>   r   r"   Tr   )r   r   r	   r   r   s    `r    
py_localfsr      sE    _%6%677881111!	   r"   c                 B    t          t                      d dd          S )Nc                     | S r   r/   r   s    r    r   zmockfs.<locals>.<lambda>        r"   Tr   )r   r   r   s    r    mockfsr      s-    {!	   r"   c                 v    t          t          t          t                                          d dd          S )Nc                     | S r   r/   r   s    r    r   zpy_mockfs.<locals>.<lambda>   r   r"   Tr   )r   r   r	   r   r   s    r    	py_mockfsr      s=    _%6%67788{!	   r"   c                 L    t          t          d          fddd          S )NTuse_mmapc                 2    | z                                   S r   r   r   s    r    r   z#localfs_with_mmap.<locals>.<lambda>   r   r"   r   r   r   s    `r    localfs_with_mmapr      s:    D)))1111!	   r"   c                 l    t          t          t          |          |d                   d dd          S )Nr   c                     | S r   r/   r   s    r    r   z!subtree_localfs.<locals>.<lambda>   r   r"   Tr   )r   r   str)r   r   r   s      r    subtree_localfsr      s;    S\\74=99{!	   r"   c              #     K   | j         j                            d           ddlm} |d         \  }}d} || d| ddt          d	
          d          }	 |                    |           n1# t          $ r$}t          j	        d| d|            Y d }~nd }~ww xY wt          ||j        dd          V  |                    |           d S )Ngcsr   GcsFileSystem
connectionpyarrow-filesystem/:httpT-   )secondstest-project-id)endpoint_overridescheme	anonymousretry_time_limit
project_idzCould not create directory in : Fr   )configpyarrowrequires
pyarrow.fsr   r   rT   OSErrorpytestskipr   __add__rV   )r   
gcs_serverr   hostportbucketr   es           r    gcsfsr      s;     N##E***((((((L)JD$"F	!**D**"2...$
 
 
B@
f @ @ @>R>>1>>????????@ ~"	      MM&s   A/ /
B9BBc              #   &  K   | j         j                            d           ddlm} |d         \  }}}}d} |||| d| ddd	          }|                    |           t          ||j        d
d
          V  |                    |           d S )Ns3r   S3FileSystemr   r   r   r   T)
access_key
secret_keyr   r   allow_bucket_creationallow_bucket_deletionFr   )	r   r   r   r   r   rT   r   r   rV   )	r   	s3_serverr   r   r   r   r   r   r   s	            r    s3fsr      s      N##D)))'''''')2<)@&D$
J"F	!**D**""
 
 
B MM&
~"	      MM&r"   c                 ^    d}t          t          ||d                   |j        dd          S )Nzpyarrow-filesystem/prefix/r   Fr   )r   r   r   )r   r   prefixs      r    subtree_s3fsr     s:    )FVT$Z00~"	   r"   a  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:PutObjectTagging",
                "s3:DeleteObject",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]

        },
        {
            "Effect": "Deny",
            "Action": [
                "s3:DeleteObject"
            ],
            "Resource": [
                "arn:aws:s3:::no-delete-bucket*"
            ]
        }
    ]
}c              #   .  K   | j         j                            d           ddlm} |d         \  }}}}| d| }d}d}	 |||||||          }
|
                    |	           t          |
|	j        d	d	
          V  |
                    |	           d S )Nazurer   AzureFileSystemr   r   r   r   account_nameaccount_keyblob_storage_authoritydfs_storage_authorityblob_storage_schemedfs_storage_schemeTr   )	r   r   r   r   r   rT   r   r   rV   )r   azure_serverr   r   r   r   r   azurite_authorityazurite_scheme	containerr   s              r    azurefsr   7  s      N##G,,,******,8,F)D$k(($((N%I	l%00A/@-;,:
< 
< 
<B MM)
 !	      MM)r"   c                     | j         j                            d           t          j                    st          j        d           ddlm} |\  }}} ||||          }t          |d dd          S )	NhdfsCannot locate libhdfsr   HadoopFileSystem)r   userc                     | S r   r/   r   s    r    r   zhdfs.<locals>.<lambda>a  r   r"   Tr   )
r   r   r   rp   have_libhdfsr   r   r   r   r   )r   hdfs_connectionr   r   r   r   r   s          r    r   r   T  s    N##F+++? -+,,,++++++&D$		$T	5	5	5B{!	   r"   c                     t          j        d          }|                    d          }t          t	          t          |                    fddd          S )Nfsspecr7   c                 2    | z                                   S r   r   r   s    r    r   z#py_fsspec_localfs.<locals>.<lambda>m  r   r"   Tr   )r   importorskip
filesystemr   r   r   r   r   r   r   s    `  r    py_fsspec_localfsr   g  sb     **F			6	"	"Bb))**1111!	   r"   c                     t          j        dd          }|j        dk    rt          j        d           |                    d          }t          t          t          |                    d dd	          S )
Nr   z0.7.5)
minversionz0.8.5z,Bug in fsspec 0.8.5 for in-memory filesystemmemoryc                     | S r   r/   r   s    r    r   z$py_fsspec_memoryfs.<locals>.<lambda>|  r   r"   Tr   )r   r   __version__r   r   r   r   r   r   s       r    py_fsspec_memoryfsr   s  s~     g>>>FW$$BCCC			8	$	$Bb))**{!	   r"   c           
   #   h  K   t          j        d          }|d         \  }}}}d}|                    ||t          d| d|                     }t	          t          |                    }|                    |           t          ||j        dd	
          V  |                    |           d S )Nr   r   r   zhttp://r   )endpoint_url)keysecretclient_kwargsFTr   )	r   r   r   r   r   r   rT   r   rV   )	r   r   r   r   r   r   r   r   r   s	            r    py_fsspec_s3fsr    s      v&&D)2<)@&D$
J"F			(?$(?(?(?(?@@@ 
 
 
B
 
mB''	(	(BMM&
~!	      MM&r"   r   zLocalFileSystem())idr   zLocalFileSystem(use_mmap=True)r   z$SubTreeFileSystem(LocalFileSystem())r   r   )r  marksr   r   r   r   r   r   r   z_MockFileSystem()r   z-PyFileSystem(ProxyHandler(LocalFileSystem()))r   z-PyFileSystem(ProxyHandler(_MockFileSystem()))r   z5PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))r   z8PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))r  z0PyFileSystem(FSSpecHandler(s3fs.S3FileSystem())))paramsc                 6    |                      | j                  S r   )getfixturevalueparamr   s    r    filesystem_configr
    s    x ""7=111r"   c                     | d         S Nr   r/   r
  s    r    r   r     s    T""r"   c                     | d         S )Nr   r/   r  s    r    r   r     s    X&&r"   c                     | d         S )Nr   r/   r  s    r    r   r     s    -..r"   c                     | d         S )Nr   r/   r  s    r    r   r     s    344r"   c                 r   t          | j        t                    sJ t          | j        t                    sJ | j        dk    sJ | j        t          j        | j                                        dz            k    sJ | j        j        }|J |	                    d           t          d          k    sJ d S )Nr   g    eA)r%   mtimer   mtime_nsintr   approx	timestamptzinfo	utcoffsetr   )	file_infor  s     r    check_mtimer    s    iox00000i(#.....""""!!##c)"+ "+ + + + + _#FD!!Yq\\111111r"   c                 *    | j         J | j        J d S r   )r  r  r  s    r    check_mtime_absentr    s%    ?"""%%%%%r"   c                 T    | j         t          |            d S t          |            d S r   )r  r  r  r  s    r    check_mtime_or_absentr    s2    9%%%%%Ir"   c                 J    | j         dk    rt          j        d           d S d S )Npy::fsspec+('s3', 's3a')zNot working with fsspec's s3fsreason)	type_namer   xfailr   s    r    skip_fsspec_s3fsr'    s2    	|111<====== 21r"   c                 J    | j         dk    rt          j        |           d S d S )Nabfsr"  )r$  r   r   )r   r#  s     r    
skip_azurer*  	  s0    	|v6"""""" r"   c                 |   ddl m} t          | t          dd           | d         \  }}}} |dd| d| d          }|                    d	           t          j        t          j        d
          5  |                    d           d d d            n# 1 swxY w Y   t          j        t          j        d          5  |	                    d           d d d            n# 1 swxY w Y   t          j        t          d          5  |                    dd           d d d            n# 1 swxY w Y   t          j        t          j        d          5  |                    d          5 }|                    d           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |                    d          5 }|                    d           d d d            n# 1 swxY w Y   t          j        t          d          5  |                    d           d d d            d S # 1 swxY w Y   d S )Nr   r   test_fs_limited_user
limited123r   r   r   )r   r   r   r   zexisting-bucket/testzBucket 'new-bucket' not foundmatchz
new-bucketzWould delete bucketzexisting-bucketzRequest ID:zexisting-bucket/test-filezexisting-bucket/test-file-copyznon-existing-bucket/test-file   testzno-delete-bucket/test-file)r   r   r
   _minio_limited_policyrT   r   raisesrp   ArrowIOErrorrV   r   rk   r}   writer`   )r   r   r   r   _r   fs          r    +test_s3fs_limited_permissions_create_bucketr7    s   ''''''y*?5|E E E .D$1	)!**D**	
 
 
B MM()))	r.M	N	N	N $ $
l###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ 
r.C	D	D	D ) )
'((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) 
wm	4	4	4 T T
02RSSST T T T T T T T T T T T T T T 
rm	<	<	<  ""#BCC 	qGGG	 	 	 	 	 	 	 	 	 	 	 	 	 	 	              
 
		;	<	< 	               
wm	4	4	4 5 5
34445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5s   4BBBC##C'*C'	D,,D03D0F&-FF&F	F&F	F&&F*-F*G((G,/G,H11H58H5c                     t          j        dt          j                  } t	          d          }|j        dk    sJ |j        dk    sJ |j        t          j	        k    sJ |j
        J t          |           t	          dt          j        dd          }|j        dk    sJ |j        dk    sJ |j        t          j        k    sJ |j
        dk    sJ |j        d	k    sJ t          |           t	          d
t          j        |           }|j        d
k    sJ |j        d
k    sJ |j        t          j        k    sJ |j
        J |j        | k    sJ |j        dk    sJ t          |           d S )NR]foo/barbarzfoo/baz.txtrG   g  `A)typerH   r  zbaz.txtl    t<|+. foo)r<  r  l    4 |+. )r   fromtimestampr   utcr   r4   	base_namer<  r   UnknownrH   r  r<   r  r  r=   r  )dtrB   s     r    test_file_info_constructorrC  2  s   		
HL	9	9BID9	!!!!>U""""9(((((9tMC&( ( (D9%%%%>Y&&&&9%%%%9=/////E 2"===D9>U""""9*****9:=/////r"   c                      t          j        t                    5  t                       d d d            d S # 1 swxY w Y   d S r   )r   r2  	TypeErrorr   r/   r"   r    'test_cannot_instantiate_base_filesystemrF  O  s    	y	!	!                   s   6::c                     t                      } t                      }t                      }|                     |           sJ |                     |          sJ t          j        t
                    5  |                     d           d d d            n# 1 swxY w Y   | | cxk    r|k    sn J | dk    sJ ||k    sJ |t                      k    sJ t          d|           t          d|           k    sJ t          d|           t          d|          k    sJ t          d|           t          d|           k    sJ d S )Nstring   /basez/other)r   r   equalsr   r2  rE  r   )fs0fs1fs2s      r    test_filesystem_equalsrO  T  s   


C


C


C::c???::c???	y	!	!  

8              #!8888#::::/######Wc**.?.M.MMMMMWc**.?.M.MMMMMWc**.?#.N.NNNNNNNs   2BBBc                     t          j        t          d          5  |                     d            d d d            n# 1 swxY w Y   | J d S )Nzgot NoneTyper.  )r   r2  rE  rK  r&  s    r    test_filesystem_equals_nonerQ  h  s    	y	7	7	7  
		$               >>>>>s   >AAc                  r   t                      } t          d|           }|j        dk    sJ |j        | k    sJ t	          |                              d          sJ t          dt                                }|j        dk    sJ |j        | k    sJ t	          |                              d          sJ d S )NrJ  z/base/zHSubTreeFileSystem(base_path=/base/, base_fs=<pyarrow._fs.LocalFileSystemz/another/base/zPSubTreeFileSystem(base_path=/another/base/, base_fs=<pyarrow._fs.LocalFileSystem)r   r   	base_pathbase_fsrepr
startswith)r   subfss     r    test_subtree_filesystemrX  o  s    Ggw//E?h&&&&=G####;;!! #I J J J J J .0A0ABBE?.....=G####;;!! #J K K K K K K Kr"   c                 (   | j                             d          d         dk    rt          j        d           |                    |           }|                    |          }t          |t                    sJ |                    |           sJ d S )N::mock"MockFileSystem is not serializabler"  )	r$  splitr   r%  dumpsloadsr%   r   rK  )r   pickle_module
serializedrestoreds       r    test_filesystem_picklingrd    s    	|$#v--@AAAA$$R((J"":..Hh
+++++??2r"   c                    | j                             d          d         dk    rt          j        d           t	          |             |d          } |d          } |d          }|                     |           |                     |          5  	 d d d            n# 1 swxY w Y   |                     |          5 }|                    d	           d d d            n# 1 swxY w Y   |                    |	                    |                     }|
                    |||g          \  }}	}
|j        t          j        k    sJ |	j        t          j        k    sJ |
j        t          j        k    sJ d S )
NrZ  r[  r\  r]  r"  	a/aa/aaa/a/bbc.txtr0  )r$  r^  r   r%  r'  rT   r}   r4  r`  r_  rC   r<  r   r=   r<   )r   r   ra  aaabbcfprc  aaa_infobb_infoc_infos              r    ,test_filesystem_is_functional_after_picklingrp    s   	|$#v--@AAAAR
&

C	BwAMM#			r	"	"                			q	!	! R
               ""=#6#6r#:#:;;H ( 6 6R| D DHgv=H.....<8=((((;(-''''''s$   B""B&)B&C$$C(+C(c                  r    t                      } | j        dk    sJ t                      } | j        dk    sJ d S )Nlocalr\  )r   r$  r   r&  s    r    test_type_namers    sD    			B<7""""			B<6!!!!!!r"   c                 <    |                      d          dk    sJ d S )Nr=  )r5   r&  s    r    test_normalize_pathru    s)     U##u,,,,,,r"   c           	          G d d          }dd |            t                      i g d t          j                    g}|D ]H}t          j        t
                    5  |                     |           d d d            n# 1 swxY w Y   Id S )Nc                       e Zd ZdS )-test_non_path_like_input_raises.<locals>.PathN)r   r   r   r/   r"   r    Pathrx    s        r"   ry     g?c                      dS )Nrz  r/   r/   r"   r    r   z1test_non_path_like_input_raises.<locals>.<lambda>  s    a r"   )tuplepathlibry  r   r2  rE  rT   )r   ry  invalid_pathsr4   s       r    test_non_path_like_input_raisesr    s            TTVVUWWb"ii\^^%M    ]9%% 	  	 MM$	  	  	  	  	  	  	  	  	  	  	  	  	  	  	    s   A==B	B	c                 D    |d          } |d          } |d          } |d          }|                      |           |                     |          5  	 d d d            n# 1 swxY w Y   |                     |          5 }|                    d           d d d            n# 1 swxY w Y   |                     ||||g          \  }}}	}
|j        |k    sJ dt          |          v sJ |j        dk    sJ | j        dk    r|j        t          j
        k    sJ n*|j        t          j        k    sJ d	t          |          v sJ |j        J t          |           |j        t          |          k    sJ |j        d
k    sJ |j        dk    sJ |j        t          j        k    sJ dt          |          v sJ |j        dk    sJ | j        dvrt#          |           |	j        t          |          k    sJ |	j        dk    sJ |	j        dk    sJ |	j        t          j        k    sJ dt          |	          v sJ |	j        dk    sJ | j        dvrt#          |	           |
j        t          |          k    sJ |
j        dk    sJ |
j        dk    sJ |
j        t          j
        k    sJ |
j        J |
j        J dt          |
          v sJ t'          |
           |                     |          }|j        |j        k    sJ |j        |j        k    sJ d S )Nrf  rg  rh  zzzr0  ri   r!  zFileType.Directoryrj  zFileType.Filer   )zpy::fsspec+memoryr!  txtrI  zFileType.NotFound)rT   r}   r4  rC   r4   rU  	extensionr$  r<  r   r>   r=   rH   r  r   r@  r<   r  r  r  )r   r   ri  rj  rk  r  rl  rm  rn  ro  zzz_info	aaa_info2s               r    test_get_file_infor    s   
&

C	BwA
&--CMM#			r	"	"                			q	!	! R
               +-*:*:CQ;L*M*M'Hgvx=CDNN""""####	|111} 111111} 22222#tH~~5555=   (###<3r77""""$$$$""""<8=((((d7mm++++<1	|LLLG;#a&&    w&&&&u$$$$;(-''''d6ll****;!	|LLLF=CHH$$$$&&&&####=H-----=   >!!!$x..0000x      %%I=IN****=IN******s$   A%%A),A)B''B+.B+c                     |d          } |d          } |d          } |d          } |d          } |d          }	 |                      |           |                     |          5  	 d d d            n# 1 swxY w Y   |                     |          5  	 d d d            n# 1 swxY w Y   |                      |           |                     |          5  	 d d d            n# 1 swxY w Y   |                      |           t          |dd	          }|j        |k    sJ |                     |          }	| j        d
k    rt          |	          dk     nt          |	          dk    sJ |	D ]}
|
j                            |          s4|
j                            |          s|
j                            |          r|
j	        t          j        k    sJ n|
j                            d                              |          s-|
j                            d                              |          r|
j	        t          j        k    sJ nt          d|
j                   t          |
           t          |d          }|                     |          }	| j        d
k    rt          |	          dk    sJ nt          |	          dk    sJ |                     |           d S # |                     |           w xY w)Nzselector-dir/zselector-dir/test_file_azselector-dir/test_file_bzselector-dir/test_dir_az#selector-dir/test_dir_a/test_file_czselector-dir/test_dir_bFT)rL   rN   r!  rI     rX   zunexpected path rN      )rT   r}   r   rK   rC   r$  lenr4   endswithr<  r   r<   rstripr=   r[   r  rV   )r   r   rK   file_afile_bdir_afile_cdir_brO   rP   rB   s              r     test_get_file_info_with_selectorr    s   vo&&HV.//FV.//FF,--EV9::FF,--E- 
h""6** 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	""6** 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	
e""6** 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	
e  %*.0 0 0 H,,,,  **<555JJ!OOOu::???? 		( 		(D	""6** Adi.@.@.H.H AI&&v..AyHM11111)""3''0077 A)""3''0077AyH$666666 !?DI!?!?@@@!$''''  E:::  **<555u::?????u::???? 	hhsy   *K- .A<0K- <B  K- B K- B*K- *B..K- 1B.2-K- C-!K- -C11K- 4C15G!K- -Lc                    t          |             |d          }t          j        t          j                  5  |                     |           d d d            n# 1 swxY w Y   |                     |           |                     |            |d          }|                     |d           |                     |           d S )Nztest-directory/zdeeply/nested/test-directory/Tr  )r'  r   r2  rp   r3  rV   rT   )r   r   ds      r    test_create_dirr  *  s     R !!A	r	'	'  
a               MM!MM!.//AMM!tM$$$MM!s   AA"Ac                    t          |             |d          } |d          }|                     |           |                     |           t          j        t
          j                  5  |                     |           d d d            n# 1 swxY w Y   t          j        t
          j                  5  |                     |           d d d            d S # 1 swxY w Y   d S )N
directory/directory/nested/)r'  rT   rV   r   r2  rp   r3  r   r   r  nds       r    test_delete_dirr  ;  sT   R|A	#	$	$BMM"MM!	r	'	'  
b              	r	'	'  
a                 s$   .BBB9CC #C c                    t          |             |d          } |d          }|                     |           |                     |           |                     |           |                     |          }|j        t
          j        k    sJ  |d          } |d          } |d          }|                     |           |                     |           |                     |          5 }|                    d           d d d            n# 1 swxY w Y   |                     |           |                     |          }|j        t
          j        k    sJ d S )Nr  r  
directory2zdirectory2/nestedzdirectory2/nested/target-file   data)	r'  rT   rV   rC   r<  r   r>   r}   r4  )r   r   r  r  dir_infor6  ss          r    $test_delete_dir_with_explicit_subdirr  I  s   
 R|A	#	$	$B MM!MM"MM!""H=H----- 	|A	#	$	$B.//AMM!MM"			q	!	! Q	               MM!""H=H-------s   1DDDc                    t          |             |d          } |d          }|                     |           |                     |           t          j        t
          j                  5  |                     |           d d d            n# 1 swxY w Y   |                     |d           t          j        t
          j                  5  |                     |           d d d            n# 1 swxY w Y   |                     |           t          j        t
          j                  5  |                     |           d d d            d S # 1 swxY w Y   d S )Nr  r  T)r\   )r'  rT   rY   r   r2  rp   r3  rV   r  s       r    test_delete_dir_contentsr  i  s   R|A	#	$	$BMM"1	r	'	'  
b              2d333	r	'	' # #
r"""# # # # # # # # # # # # # # #MM!	r	'	'  
a                 s6   .BBBC22C69C60EEEc                 `   | d         }| d         } |d          } |d          }|                     |           t          j        t          j                  5  |                    d           d d d            n# 1 swxY w Y   t          j        t          j                  5  |                    d           d d d            n# 1 swxY w Y   t          j        t          j                  5  |                    d           d d d            n# 1 swxY w Y   |                    dd	           |                    dd	           |                    dd	           t          j        t          j                  5  |                    |           d d d            d S # 1 swxY w Y   d S )
Nr   r   r  r  r  rX   //Taccept_root_dir)rT   r   r2  rp   ArrowInvalidrY   r3  rV   )r   r   r   r  r  s        r    _check_root_dir_contentsr  {  s   	BHF|A	#	$	$BMM"	r	'	' # #
r"""# # # # # # # # # # # # # # #	r	'	' $ $
s###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $	r	'	' % %
t$$$% % % % % % % % % % % % % % % 2t44435554666	r	'	'  
a                 sH   A<<B B %CCC0DDD F##F'*F'c                 B    t          |            t          |           d S r   )r  )r   r   s     r    test_delete_root_dir_contentsr    s$    V$$$Y'''''r"   c                     |d          } |d          }|                      |          5  	 d d d            n# 1 swxY w Y   |                     ||           |                     |           |                     |           d S )Nztest-copy-source-fileztest-copy-target-file)r}   rk   r`   r   r   r  ts       r    test_copy_filer    s    &''A&''A			q	!	!                 LLANN1NN1s   :>>c                 
   t          | d            |d          } |d          }|                     |           |rx|                     ||           t          j        t
          j                  5  |                     |           d d d            n# 1 swxY w Y   |                     |           d S t          j        t
          j                  5  |                     ||           d d d            d S # 1 swxY w Y   d S )N.Not implemented yet in for Azure. See GH-40025zsource-dir/ztarget-dir/)r*  rT   rg   r   r2  rp   r3  rV   )r   r   r   r  r  s        r    test_move_directoryr    sv   rCDDD 	}A}AMM! 
1]2?++ 	 	MM!	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
a]2?++ 	 	GGAqMMM	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s$   2BBBC88C<?C<c                    t          |            t          | d            |d          } |d          }|                     |          5  	 d d d            n# 1 swxY w Y   |                     ||           t	          j        t          j                  5  |                     |           d d d            n# 1 swxY w Y   |                     |           d S )Nr  ztest-move-source-fileztest-move-target-file)	r'  r*  r}   rg   r   r2  rp   r3  r`   r  s       r    test_move_filer    sU    R rCDDD&''A&''A			q	!	!                 GGAqMMM	r	'	'  
q              NN1s$   AA AB::B>B>c                 <    |d          }|                      |          5  	 d d d            n# 1 swxY w Y   |                     |           t          j        t          j                  5  |                     |           d d d            n# 1 swxY w Y    |d          }|                     |            |d          }|                      |          5 }|                    d           d d d            n# 1 swxY w Y   |                     |           d S )Nztest-delete-target-fileztest-delete-nestedztest-delete-nested/target-filer  )	r}   r`   r   r2  rp   r3  rT   r4  rV   )r   r   r   r  r6  r  s         r    test_delete_filer    s   ())A			q	!	!                 NN1	r	'	'  
q               	#$$AMM!/00A			q	!	! Q	               MM!s0   /33-BBBC<<D D c                     | S r   r/   )vs    r    identityr    s    Hr"   )compressionbuffer_size
compressor@   gzip   c                 R    |d          }ddz  }|                      |          5 }|                     ||                     d d d            n# 1 swxY w Y   |                     |||          5 }|                                }d d d            n# 1 swxY w Y   ||k    sJ d S )Nzopen-input-streams   some data for reading
   )r}   r4  rt   read)	r   r   r  r  r  r   rs   r  results	            r    test_open_input_streamr    s,    	"##A%+D			q	!	! "Q	

4  !!!" " " " " " " " " " " " " " " 
		ak	:	: a               T>>>>>>s#   AAA3BBBc                 ,    |d          }ddz  }|                      |          5 }|                    |           d d d            n# 1 swxY w Y   t          d          dz  }|                     |          5 }|                                }d d d            n# 1 swxY w Y   ||k    sJ |                     |          5 }|                    |           |                                }d d d            n# 1 swxY w Y   |||d          k    sJ d S )Nzopen-input-file	   some data   r  )r}   r4  r  rv   r  seek)r   r   r   rs   r  	read_fromr6  r  s           r    test_open_input_filer    s    !!A$D			q	!	! Q	               L!!C'I			A		 !              T>>>>			A		 !	y               T)**%%%%%%%s5   AAA:BB"B*C99C= C=c                      |d          }t          j        t                    5  |                     |           d d d            d S # 1 swxY w Y   d S )Nzopen-input-stream-not-found)r   r2  rM   rt   )r   r   r   s      r     test_open_input_stream_not_foundr    s    ,--A	(	)	)    
Q                                   s   AAA)r  r  decompressorc                 `    |d          }ddz  }|                      |||          5 }|                    |           d d d            n# 1 swxY w Y   |                     |||          5 }|                    t	          |                    |k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nzopen-output-streams   some data for writingr  )r}   r4  rt   r  r  )r   r   r  r  r  r   rs   r6  s           r    test_open_output_streamr    s8    	#$$A#d*D			q+{	;	; q	               
		ak	:	: )avvc$ii  D((((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s#   A

AA,)B##B'*B')r  r  r  r  c                     |d          } |d          }|                      |          5 }	|	                    |           d d d            n# 1 swxY w Y   |r|                     |||          5 }
|
                    d           d d d            n# 1 swxY w Y   |                     |          5 }
|
                                }d d d            n# 1 swxY w Y    ||          }|dk    sJ d S t          j        t          j                  5  |                     |||           d d d            d S # 1 swxY w Y   d S )Nzopen-append-streams   already existing)r  r  s   
newly addeds   already existing
newly added)	r}   r4  r   rt   r  r   r2  rp   ArrowNotImplementedError)r   r   r  r  r  r  r   r   initialr  r6  r  s               r    test_open_append_streamr  (  sg    	#$$Aj,--G			q	!	! Q	                ;""1+/: # < < 	&?@GG$%%%	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& !!!$$ 	VVXXF	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 f%%9999999]2677 	; 	;!!!.9 " ; ; ;	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	;sG   AAA3BBB5CCCD::D>D>c                 2    |d          }ddi}d}|                      ||          5 }|                    |           d d d            n# 1 swxY w Y   |                     |          5 }|                                |k    sJ |                                }d d d            n# 1 swxY w Y   | j        dv s	d| j        v rLt          | d           |                                D ]%\  }}||         |                                k    sJ &d S |i k    sJ d S )	Nzopen-output-stream-metadataContent-Typezx-pyarrow/testr  r|   )r   r   r)  r\  zWAzure filesystem currently only returns system metadata not user metadata. See GH-40026)	r}   r4  rt   r  r|   r$  r*  itemsro   )	r   r   r   r|   rs   r6  got_metadatakr  s	            r     test_open_output_stream_metadatar  J  s   ,--A 01HD			q8		4	4 	               
		a	 	  $Avvxx4zz||$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ 
|,,,",0F0F %	& 	& 	& NN$$ 	1 	1DAq?ahhjj00000	1 	1 r!!!!!!s#   AAA+/B&&B*-B*c                      t          d           t          j        t                    5  t          d           d d d            d S # 1 swxY w Y   d S )NFr   )xxx)r   r   r2  rE  r/   r"   r    test_localfs_optionsr  a  s    U####	y	!	! # #E""""# # # # # # # # # # # # # # # # # #s   AAAc                 Z   | d         }t                      5  |                    d           d d d            n# 1 swxY w Y   t                      5  |                    d           d d d            n# 1 swxY w Y   t                      5  |                    dd           d d d            n# 1 swxY w Y   t                      5  |                    d           d d d            n# 1 swxY w Y   t                      5  |                    d           d d d            n# 1 swxY w Y   t                      5  |                    dd           d d d            n# 1 swxY w Y   t                      5  |                    dd           d d d            d S # 1 swxY w Y   d S )Nr   z/non/existent/filez/non/existent/dirFr  z/non/existentz/xxx)r   rt   r}   rT   rV   r`   rg   rk   )r   r   s     r    test_localfs_errorsr  i  sR    
B		 	  3 3
12223 3 3 3 3 3 3 3 3 3 3 3 3 3 3		 	  4 4
23334 4 4 4 4 4 4 4 4 4 4 4 4 4 4		 	  < <
)U;;;< < < < < < < < < < < < < < <		 	  + +
)***+ + + + + + + + + + + + + + +		 	  , ,
*+++, , , , , , , , , , , , , , ,		 	  ) )
((() ) ) ) ) ) ) ) ) ) ) ) ) ) )		 	  . .
_f---. . . . . . . . . . . . . . . . . .sy   9= =A44A8;A8B11B58B5
C,,C03C0D''D+.D+ E##E'*E'<F  F$'F$c                    | d         }t          j        t                    }|j        }|                    |                                |                                g          \  }}|j        |                                j        k    sJ |j	        |                                j
        k    sJ t          |           |j	        |                                j
        k    sJ t          |           d S r  )r}  ry  __file__parentrC   r   rH   statst_sizer  st_mtime_nsr  )r   r   	file_pathdir_pathr  r  s         r    test_localfs_file_infor  }  s    	BX&&IH,,i.@.@.B.B.6.?.?.A.A.C D DY>Y^^--55555!1!1!=====	 ;;;;;r"   c                    t          j        dt          j                  }t	          |          }|                    d          5  	 d d d            n# 1 swxY w Y   |                    dg          \  }|j        |k    sJ d S )Nr9  r=  )r   r>  r   r?  r   r}   rC   r  )r   rB  r   rB   s       r    test_mockfs_mtime_roundtripr    s    		
HL	9	9B			B			u	%	%                ug&&FT:s   AAAc           	         ddl m} t          j                    } |dd|dddd	          }t	          ||          sJ |j        dk    sJ |j        dk    sJ |                     |                     |                    |k    sJ  |            }t	          ||          sJ |                     |                     |                    |k    sJ  |d
          }t	          ||          sJ |                     |                     |                    |k    sJ  |ddd          }t	          ||          sJ |                     |                     |                    |k    sJ t          j
        t                    5   |d           d d d            n# 1 swxY w Y   t          j
        t                    5   |d
d           d d d            n# 1 swxY w Y   t          j
        t                    5   |d
d           d d d            n# 1 swxY w Y   t          j
        t                    5   |t          j                               d d d            d S # 1 swxY w Y   d S )Nr   r   abczservice_account@apachezus-west2httpslocalhost:8999r   )access_tokentarget_service_accountcredential_token_expirationdefault_bucket_locationr   r   r   Tr   authenticated-read
text/plainACLr  default_metadataaccess)r  r  )r   r  acct)r   r  )r  )r   r   r   nowr%   r  r   r`  r_  r   r2  r[   )ra  r   rB  r   s       r    test_gcs_optionsr    s   ((((((	B	E.F35/9%9I"3
5 
5 
5B b-(((((%3333=-----}22266772====	Bb-(((((}22266772====		&	&	&Bb-(((((}22266772====	0D9E)G )G 
H 
H 
HBb-(((((}22266772====	z	"	" - -8,,,,- - - - - - - - - - - - - - -	z	"	" = =8<<<<= = = = = = = = = = = = = = =	z	"	" E EVDDDDE E E E E E E E E E E E E E E	z	"	" B B(,..AAAAB B B B B B B B B B B B B B B B B BsH   FF!$F!GG"G?HH H=I((I,/I,c                    ddl m}m}m}m}  |dddddd	          }t          ||          sJ |j        dk    sJ |                     |                     |                    |k    sJ  |d
ddd          }t          ||          sJ |                     |                     |                    |k    sJ  | |d                    }t          ||          sJ  | |d                    }t          ||          sJ  |d
          }t          ||          sJ |                     |                     |                    |k    sJ ||k    sJ  |d          }t          ||          sJ |                     |                     |                    |k    sJ  |d          }t          ||          sJ |                     |                     |                    |k    sJ  |dddd          }t          ||          sJ |                     |                     |                    |k    sJ ||k    sJ  |d          }t          ||          sJ |                     |                     |                    |k    sJ |                     |                     |                     |            k    sJ  |dd          }t          ||          sJ |                     |                     |                    |k    sJ  |ddd          }t          ||          sJ |                     |                     |                    |k    sJ  |dd          }t          ||          sJ |                     |                     |                    |k    sJ  |dd          }t          ||          sJ |                     |                     |                    |k    sJ ||k    sJ  |dd           }t          ||          sJ |                     |                     |                    |k    sJ  |d!"          }t          ||          sJ |                     |                     |                    |k    sJ | |d#"          k    sJ | |            k    sJ t          j
        t                    5   |d$           d d d            n# 1 swxY w Y   t          j
        t                    5   |d%           d d d            n# 1 swxY w Y   t          j
        t                    5   |dd&           d d d            n# 1 swxY w Y   t          j
        t                    5   |dd'           d d d            n# 1 swxY w Y   t          j
        t                    5   |ddd()           d d d            n# 1 swxY w Y   t          j
        t                    5   |ddd*           d d d            n# 1 swxY w Y   t          j
        t                    5   |d(d+           d d d            n# 1 swxY w Y   t          j
        t                    5   |d,d-g.           d d d            n# 1 swxY w Y   t          j
        t                    5   | |                       d d d            d S # 1 swxY w Y   d S )/Nr   )AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyr   S3RetryStrategyr  r  token	us-east-2r  r  )r   r   session_tokenregionr   r   rolesessionr  d   )role_arnsession_nameexternal_idload_frequencyr  )max_attempts)retry_strategy)r  Tr  )background_writesr  r  r  )r  r  )allow_delayed_open)r   r   )r   r   )check_directory_existence_before_creationg      ?g      ?)request_timeoutconnect_timeout)r   force_virtual_addressingzca.pem)tls_ca_file_pathzother_ca.pem)r   )r   )r   r  )r   r  arn)r   r   r  )r   r   r   )r  r   r=  r;  r  )r   r  r  r   r  r%   r  r`  r_  r   r2  r[   )ra  r  r  r   r  r   rN  s          r    test_s3_optionsr    s
   - - - - - - - - - - - - 
h$+K$8H
J 
J 
JB b,'''''9####}22266772====	vI"&s
< 
< 
<Bb,'''''}22266772==== 
11qAAA
C 
C 
CBb,'''''	00a@@@
B 
B 
BBb,'''''
,
'
'
'Cc<(((((}2237788C????"9999		%	%	%Bb,'''''}22266772====		-	-	-Bb,'''''}22266772====
,0D9E)G )GH H HC c<(((((}2237788C????"9999		.	.	.Bb,'''''}22266772====}2226677<<>>IIII	D	M	M	MBb,'''''}22266772====	D@D
F 
F 
FBb,'''''}22266772====	c4	@	@	@Bb,'''''}22266772====
,tS
A
A
ACc<(((((}2237788C????"9999	(8SW	X	X	XBb,'''''}22266772====	x	0	0	0Bb,'''''}22266772====~>>>>>>>	z	"	" * *))))* * * * * * * * * * * * * * *	z	"	" * *))))* * * * * * * * * * * * * * *	z	"	" A A@@@@A A A A A A A A A A A A A A A	z	"	" A A@@@@A A A A A A A A A A A A A A A	z	"	" 
 
Hu	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z	"	" 
 
H	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z	"	" 5 5et44445 5 5 5 5 5 5 5 5 5 5 5 5 5 5	z	"	" 6 6uen55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6	z	"	" 7 7OO$5$566667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7s   !S::S>S>T77T;>T;U55U9<U9V33V7:V7W22W69W6X11X58X5Y//Y36Y3Z..Z25Z2[44[8;[8c                    ddl m} dddd}d}ddd	d}d
} ||          }t          ||          sJ |                    |                    |                    |k    sJ  ||          }t          ||          sJ |                    |                    |                    |k    sJ  ||          }t          ||          sJ |                    |                    |                    |k    sJ  ||          }t          ||          sJ |                    |                    |                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } ||          }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } |            }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } |            }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } |            }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ  ||          } |            }	||	k    sJ |                    |                    |                    |	k    sJ |                    |                    |	                    |k    sJ t          j        t                    5   |d           d d d            n# 1 swxY w Y   t          j        t                    5   |ddd           d d d            n# 1 swxY w Y   t          j        t                    5   |ddd           d d d            n# 1 swxY w Y   t          j        t                    5   |ddd           d d d            n# 1 swxY w Y   t          j        t          j
                  5   |d           d d d            n# 1 swxY w Y   t          j        t          j
                  5   |dddd           d d d            d S # 1 swxY w Y   d S )Nr   r   r   	localhosti'#  )r   r   r   zhttp://localhost:8999r  i  zhttps://localhost:8080)proxy_options)r   r  #  r  )r   r   )r   r   )r   r   zhttpsB://localhost:9000httpA)r   r   r%   r`  r_  r   r2  rE  KeyErrorrp   r  )
monkeypatchra  r   proxy_opts_1_dictproxy_opts_1_strproxy_opts_2_dictproxy_opts_2_strr   rM  rN  s
             r    test_s3_proxy_optionsr%  #  s   '''''' $*;MM.#*KNN/ 
$5	6	6	6Bb,'''''}22266772====	$5	6	6	6Bb,'''''}22266772==== 
$4	5	5	5Bb,'''''}22266772====	$4	5	5	5Bb,'''''}22266772==== ,%6
7
7
7C
,%6
7
7
7C#::::}2237788C????}2237788C????
,%6
7
7
7C
,%6
7
7
7C#::::}2237788C????}2237788C???? ,%5
6
6
6C
,%5
6
6
6C#::::}2237788C????}2237788C????
,%5
6
6
6C
,%5
6
6
6C#::::}2237788C????}2237788C???? ,%6
7
7
7C
,%5
6
6
6C#::::}2237788C????}2237788C????
,%6
7
7
7C
,%5
6
6
6C#::::}2237788C????}2237788C???? ,%6
7
7
7C
,%6
7
7
7C#::::}2237788C????}2237788C????
,%6
7
7
7C
,%5
6
6
6C#::::}2237788C????}2237788C????
,%5
6
6
6C
,%6
7
7
7C#::::}2237788C????}2237788C????
,%5
6
6
6C
,%5
6
6
6C#::::}2237788C????}2237788C???? ,%6
7
7
7C
,..C#::::}2237788C????}2237788C????
,%5
6
6
6C
,..C#::::}2237788C????}2237788C????
,%6
7
7
7C
,..C#::::}2237788C????}2237788C????
,%5
6
6
6C
,..C#::::}2237788C????}2237788C???? 
y	!	! @ @#>????@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ 
x	 	  H HK#F#FGGGGH H H H H H H H H H H H H H H 
x	 	  F Fgt#D#DEEEEF F F F F F F F F F F F F F F 
x	 	  L Lfk#J#JKKKKL L L L L L L L L L L L L L L 
r	'	' > >#<====> > > > > > > > > > > > > > > 
r	'	' 3 3g{,0$2 $2 	3 	3 	3 	33 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3sl    `99`= `=a99a= a=b99b= b=c99c= c="d;;d?d?$ff	fc                  4   ddl m}   | dd          }d}t          j        t          |          5 }|                    d           d d d            n# 1 swxY w Y   d	t          |j                  v rd S  | d
d          }|                    d           d S )Nr   r   z
eu-north-1T)r  r   zWhen getting information for bucket 'arrow-datasets': AWS Error UNKNOWN \(HTTP status 301\) during HeadBucket operation: No response body. Looks like the configured region is 'eu-north-1' while the bucket is located in 'us-east-1'.|NETWORK_CONNECTIONr.  arrow-datasetsNETWORK_CONNECTION	us-east-1)r   r   r   r2  r   rC   r   r   )r   r   msgexcs       r    test_s3fs_wrong_regionr,    s    '''''' 
\T	:	:	:B!C
 
wc	*	*	* +c
)***+ + + + + + + + + + + + + + + s39~~--	[D	9	9	9B%&&&&&s   AAAc                    ddl m}  |d          }t          ||          sJ |                     |                     |                    |k    sJ  |dd          }t          ||          sJ |                     |                     |                    |k    sJ ||k    sJ  |ddd	d
dd          }t          ||          sJ |                     |                     |                    |k    sJ ||k    sJ  |dd          }t          ||          sJ |                     |                     |                    |k    sJ ||k    sJ  |dddd          }t          ||          sJ |                     |                     |                    |k    sJ ||k    sJ  |dd          }t          ||          sJ |                     |                     |                    |k    sJ ||k    sJ t          j        t          d          5   |dd           d d d            n# 1 swxY w Y   t          j        t          d          5   |dd           d d d            n# 1 swxY w Y   d}t          j        t          |          5   |ddd           d d d            n# 1 swxY w Y   t          j        t          d          5   |ddd           d d d            n# 1 swxY w Y   t          j        t          |          5   |ddd           d d d            n# 1 swxY w Y   t          j        t                    5   |ddd           d d d            n# 1 swxY w Y   t          j        t                    5   |             d d d            d S # 1 swxY w Y   d S )Nr   r   zfake-account-name)r   fakeaccountkey)r   r   zfake-accountfakeaccountzfake-blob-authorityzfake-dfs-authorityr  r   fakesastoken)r   	sas_tokenzfake-tenant-idzfake-client-idzfake-client-secret)r   	tenant_id	client_idclient_secret)r   r3  zclient_id must be specifiedr.  )r   r2  )r   r4  zInvalid Azure credential configuration: For ManagedIdentityCredential, provide only client_id. For ClientSecretCredential, provide tenant_id, client_id, and client_secret.)r   r3  r4  )r   r2  r4  )r   r2  r3  )r   r   r1  )	r   r   r%   r`  r_  r   r2  r[   rE  )	ra  r   rM  rN  fs3fs4fs5fs6invalid_msgs	            r    test_azurefs_optionsr:    st   ******
/':
;
;
;Cc?+++++}2237788C????
/':&68 8 8Cc?+++++}2237788C????#::::
/~=1F0D.5-4	6 6 6C
 c?+++++}2237788C????#::::
/':$24 4 4Cc?+++++}2237788C????#::::
/(""*	  C c?+++++}2237788C????#::::
/("  C c?+++++}2237788C????#::::	z)F	G	G	G 
 
,&	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z)F	G	G	G 
 
,.	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
	W  
z	5	5	5 
 
,&.	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z)F	G	G	G 
 
,&.	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z	5	5	5 
 
,&&	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z	"	" 2 2%8m"0	2 	2 	2 	22 2 2 2 2 2 2 2 2 2 2 2 2 2 2 
y	!	!                   s~   H77H;>H;I77I;>I;J::J>J> K;;K?K?!L<<M M  M;;M?M?N77N;>N;c                    ddl m} t          j                    st	          j        d           | \  }}}d}d}d}d}	 |||d|||	          }
|                    |	                    ||d|||                    }|                    |	                    ||d
|||                    }|                    |	                    ||d
|dz   ||                    } |||          }|                    d| d|           } |||d          } |||dd          } |||dt          j	        d                    } |||dd          } |||ddddi          }|
|k    sJ ||k    sJ ||k    sJ ||k    sJ ||k    sJ ||k    sJ ||k    sJ ||k    sJ ||k    sJ ||k    sJ ||k    sJ ||k    sJ t	          j
        t                    5   |             d d d            n# 1 swxY w Y   t	          j
        t                    5  |                    d           d d d            n# 1 swxY w Y   |
||||||||||fD ]0}|                    |                    |                    |k    sJ 1| \  }}} ||||          }|                    t          d                    sJ |                    d| d| d|           }|                    t          d                    sJ d S )Nr   r   r      i   i   zIhdfs://{}:{}/?user={}&replication={}&buffer_size={}&default_block_size={}libhdfs)r   replicationr  default_block_sizemerz  zhdfs://r   	localuser)r   
cache_path)r   kerb_ticketcache_path2
hdfs_tokenabcd)r   rC  
extra_confr  rX   z/?user=)r   r   rp   r   r   r   from_uriformatr}  ry  r2  rE  r`  r_  rC   r   )r   ra  r   r   r   r   r>  r  r?  urihdfs1hdfs2hdfs3hdfs4hdfs5hdfs6hdfs7hdfs8hdfs9hdfs10hdfs11r   r   s                          r    test_hdfs_optionsrV  &  s   ++++++? -+,,,&D$KK$$C T4i)4+0BD D DE %%cjjdI{K9K' '  E %%cjjdD+{4F' '  E %%cjjdD+/;8J' '  E T4((E%%&=&=&=t&=&=>>ET4k:::ET4k)57 7 7ET4k)0l)C)CE E EEdD{*79 9 9FdD{*6*6)?A A AF E>>>>E>>>>E>>>>E>>>>E>>>>E>>>>E>>>>E>>>>E>>>>E>>>>U????U????	y	!	!                	y	!	! % %!!!$$$% % % % % % % % % % % % % % % eUE5%ff& B B""=#6#6r#:#:;;rAAAAA&D$D$T222Dl3//00000$$,$,,,,d,, D l3//0000000s$   G--G14G1H33H7:H7)rJ  expected_klassexpected_pathzmock:r  zmock:foo/barr:  zmock:/foo/barzmock:///foo/barzmock:///some%20path/%C3%A9u   some path/éfile:/rX   zfile:///zfile:/foo/barz/foo/barzfile:///foo/barzfile:///some%20path/%C3%A9u   /some path/éu   /some path/%20éc                 h    t          j        |           \  }}t          ||          sJ ||k    sJ d S r   )r   rH  r%   )rJ  rW  rX  r   r4   s        r    test_filesystem_from_urir[  n  sE    & "3''HBb.)))))=      r"   c                  h   t          j        d          \  } }t          | t                    sJ |dk    sJ t          j        d          \  } }t          | t                    sJ |dk    sJ t                                          d          \  } }t          | t                    sJ |dk    sJ d S )NrY  rX   )rJ  )r   rH  r%   r   )r   r4   s     r     test_filesystem_from_uri_callingr]    s    "8,,HBb/*****3;;;; "x000HBb/*****3;;;;   ))h)77HBb/*****3;;;;;;r"   r4   c                    t          j        |           }t          j        |          \  }} t	          |t
                    sJ | |                                                                                                k    sJ d S r   )	r}  ry  r   rH  r%   r   resolveabsoluter   )r4   r   r   s      r     test_filesystem_from_path_objectra    st    
 	TA"1%%HBb/*****199;;''))2244444444r"   c           	      N   ddl m} | d         \  }}}}d| d| d| d| d	}t          j        |          \  }}t	          ||          sJ |dk    sJ |                    |           |                    |g          \  }	|	j        |k    sJ |	j        t          j
        k    sJ d S )	Nr   r   r   s3://r   z0@mybucket/foo/bar?scheme=http&endpoint_override=z&allow_bucket_creation=Truemybucket/foo/bar)r   r   r   rH  r%   rT   rC   r4   r<  r   r=   )
r   r   r   r   r   r   rJ  r   r4   rB   s
             r    test_filesystem_from_uri_s3re    s   '''''')2<)@&D$
JF* F Fz F F!F F$(F F FC "3''HBb,'''''%%%%%MM$tf%%FT99*******r"   c                 J   ddl m} | d         \  }}dd| d| dz   dz   }t          j        |          \  }}t	          ||          sJ |d	k    sJ |                    |           |                    |g          \  }|j        |k    sJ |j        t          j
        k    sJ d S )
Nr   r   r   zgs://anonymous@z/mybucket/foo/bar?scheme=http&endpoint_override=r   &z0retry_limit_seconds=5&project_id=test-project-idrd  )r   r   r   rH  r%   rT   rC   r4   r<  r   r=   )r   r   r   r   rJ  r   r4   rB   s           r    test_filesystem_from_uri_gcsrh    s    ((((((L)JD$KTKKDKKKL=>C "3''HBb-(((((%%%%%MM$tf%%FT99*******r"   c                      t                      } t          |           }t          |t                    sJ |j        dk    sJ |j        | u sJ t          j        t                    5  t          d            d d d            d S # 1 swxY w Y   d S Nz	py::dummy)r   r   r%   r$  handlerr   r2  rE  )rk  r   s     r    test_py_filesystemrl    s    nnG	g		Bb,'''''<;&&&&:    	y	!	!  T                 s   &BB
Bc                  |   t          d          } t          d          }t          d          }t          |           }t          |           }t          |          }t          |          }||usJ ||usJ ||usJ ||k    sJ ||k    sJ ||k    sJ |t                      k    sJ |t                      k    sJ d S )Nrz  r<  )r   r   r   r?   )handler1handler2handler3rM  rN  r5  r6  s          r    test_py_filesystem_equalityrq    s    AHAHAH
x
 
 C
x
 
 C
x
 
 C
x
 
 Cc>>>>c>>>>c>>>>#::::#::::#::::/######&((??????r"   c                    t                      }t          |          }|                     |          }|                     |          }t	          |t
                    sJ ||k    sJ |j        |k    sJ |j        dk    sJ d S rj  )r   r   r_  r`  r%   r   rk  r$  )ra  rk  r   rb  rc  s        r    test_py_filesystem_picklingrs    s    nnG	g		B$$R((J"":..Hh
+++++r>>>>w&&&&,,,,,,r"   c                     t                      } t          |           }t          |t                    sJ t          j        |           }d }  |            J d } |            J t                      } t          |           }t          j        |           }d }  |            |j        u sJ  |            J d } |            J d S r   )r   r   r%   weakrefrefrk  )rk  r   wrs      r    test_py_filesystem_lifetimerx    s    nnG	g		Bb,'''''	W		BG244	B244<<< nnG	g		B	W		BG2442:244	B244<<<<<r"   c                     t                      } t          |           }|                    dg          \  }|j        dk    sJ |j        t
          j        k    sJ |                    dg          \  }|j        dk    sJ |j        t
          j        k    sJ |                    dg          \  }|j        dk    sJ |j        t
          j        k    sJ t          j
        t                    5  |                    dg           d d d            n# 1 swxY w Y   t          j
        t                    5  |                    dg           d d d            d S # 1 swxY w Y   d S )Nzsome/dirz	some/filer9   r:   r  )r   r   rC   r4   r<  r   r=   r<   r>   r   r2  rE  r@   )rk  r   rB   s      r     test_py_filesystem_get_file_inforz  	  s   nnG	g		Bzl++FT9
""""9*****{m,,FT9####9%%%%zl++FT9
""""9)))))	y	!	! & &
)%%%& & & & & & & & & & & & & & & 
w		 " "
%!!!" " " " " " " " " " " " " " " " " "s$   +DDD2EEEc                     t                      } t          |           }t          d          }|                    |          }t	          |          dk    sJ |d         j        dk    sJ |d         j        t          j        k    sJ |d         j	        dk    sJ |d         j        dk    sJ |d         j        t          j
        k    sJ |d         j	        J t          dd	
          }|                    |          }t	          |          dk    sJ |d         j        dk    sJ |d         j        dk    sJ |d         j        dk    sJ t          d          }t          j        t                    5  |                    |           d d d            n# 1 swxY w Y   t          dd	          }|                    |          g k    sJ d S )NrE   )rK   r<  r   rF   rG   rz  rI   T)rK   rN   r  rJ   r9   )rK   rL   )r   r   r   rC   r  r4   r<  r   r<   rH   r=   r   r2  rM   )rk  r   rO   rP   s       r    )test_py_filesystem_get_file_info_selectorr|     s   nnG	g		BY///HX&&Eu::????8=O++++8=HM))))8=C8=-----8=H.....8=   Y$???HX&&Eu::????8=O++++8=-----8=33333Z000H	(	)	) # #
"""# # # # # # # # # # # # # # # ZFFFHH%%++++++s   7FF Fc                     t                      } t          |           }|                    dd           |                    dd           t          j        t
                    5  |                    d           d d d            n# 1 swxY w Y   |                    d           |                    d           d	D ]_}t          j        t                    5  |                    |           d d d            n# 1 swxY w Y   |                    |d
           `|	                    d           |
                    dd           |                    dd           d S )NrN   Tr  rS   FfoobarrV   rY   )r  rX   r  r  r`   rb   rc   ri   rj   )r   r   rT   r   r2  r@   rV   rY   r[   r`   rg   rk   )rk  r   r4   s      r    test_py_filesystem_opsr  =  s   nnG	g		BMM+M...MM/UM333	w		    
h                              MM,0111 ; ;]:&& 	) 	)""4(((	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	)
tT::::NN=!!!GGK###LL!>22222s$   %BBBC;;C?	C?	c                  Z   t          t                                } |                     d          5 }|                                dk    sJ 	 d d d            n# 1 swxY w Y   t	          j        t                    5  |                     d           d d d            d S # 1 swxY w Y   d S )Nsomefiles   somefile:input_streamr9   )r   r   rt   r  r   r2  rM   r   r6  s     r    test_py_open_input_streamr  Q  s   	lnn	%	%B			j	)	) 4Qvvxx3333334 4 4 4 4 4 4 4 4 4 4 4 4 4 4	(	)	) ) )
Z((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )#   AA A=B  B$'B$c                  Z   t          t                                } |                     d          5 }|                                dk    sJ 	 d d d            n# 1 swxY w Y   t	          j        t                    5  |                     d           d d d            d S # 1 swxY w Y   d S )Nr  s   somefile:input_filer9   )r   r   rv   r  r   r2  rM   r  s     r    test_py_open_input_filer  Z  s   	lnn	%	%B			J	'	' 21vvxx1111112 2 2 2 2 2 2 2 2 2 2 2 2 2 2	(	)	) ' '
:&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 'r  c                      t          t                                } |                     d          5 }|                    d           d d d            d S # 1 swxY w Y   d S Nr  r  )r   r   r}   r4  r  s     r    test_py_open_output_streamr  c      	lnn	%	%B			z	*	* a	                    AAAc                      t          t                                } |                     d          5 }|                    d           d d d            d S # 1 swxY w Y   d S r  )r   r   r   r4  r  s     r    test_py_open_append_streamr  j  r  r  c                     ddl m}  t          j                            d          pd} | d          }|j        |k    sJ  | dd          }|                    t          d                    }t          |          dk    sJ d	}|	                    |          5 }|
                                }d
|v sJ |d         dk    sJ |d         dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   r   PYARROW_TEST_S3_REGIONr)  Tr  )r   r  zarrow-datasets/nyc-taxiz8arrow-datasets/nyc-taxi/year=2019/month=6/part-0.parquetr  zLast-Modifieds   2025-11-26T10:28:55ZETags%   "4c6a76826a695c6ac61592bc30cda3df-16")r   r   osenvirongetr  rC   r   r  rt   r|   )r   default_regionr   entriesr   r6  mds          r    test_s3_real_awsr  q  s    ('''''jnn%=>> "! 		%	%	%B9&&&&	[	9	9	9B|! #  # $ $Gw<<!
DC			c	"	" FaZZ\\####/"&===== &zEEEEEEF F F F F F F F F F F F F F F F F Fs   7C!!C%(C%c                     t          j        d          \  } }| j        dk    sJ |                     |          5 }d|                    d          v sJ 	 d d d            n# 1 swxY w Y   t          j        d          \  } }| j        dk    sJ t          j        t          d          5  t          j        d	           d d d            n# 1 swxY w Y   t          j        d
          \  } }| j        dk    sJ t          j        d          \  } }|                     |          }t          j        t                    5  |	                                 d d d            d S # 1 swxY w Y   d S )Nzs3://mf-nwp-models/README.txt	eu-west-1s&   Meteo-France Atmospheric models on AWS2   z.s3://mf-nwp-models/README.txt?region=us-east-2r  zBucket '.*' not foundr.  z s3://x-arrow..nonexistent-bucketz0s3://x-arrow-nonexistent-bucket?region=us-east-3z	us-east-3zMs3://x-arrow-nonexistent-bucket/T.md?region=us-east-2&allow_delayed_open=true)
r   rH  r  rt   r  r   r2  r@   r}   close)r   r4   r6  streams       r    !test_s3_real_aws_region_selectionr    sW    "#BCCHB9####			d	#	# Gq8AFF2JJFFFFFG G G G G G G G G G G G G G G "8: :HB9#### 
w&=	>	>	> @ @>???@ @ @ @ @ @ @ @ @ @ @ @ @ @ @"#UVVHB9####
 "WY YHB""4((F	w		                   s5   A!!A%(A%+CCC E""E&)E&c                  P   ddl m}   | d          dk    sJ  | d          dk    sJ t          j        t          d          5   | d	           d d d            n# 1 swxY w Y   t          j        t          d          5   | d
           d d d            d S # 1 swxY w Y   d S )Nr   resolve_s3_regionr'  r)  zmf-nwp-modelsr  zNot a valid bucket namer.  r:  z	s3:bucket)r   r  r   r2  r[   r  s    r    test_resolve_s3_regionr    sT   ,,,,,,-..+====_--<<<<	z)B	C	C	C % %)$$$% % % % % % % % % % % % % % %	z)B	C	C	C ' '+&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's$   AA #A BB"Bc           
      ^   |d         }|d         } |d          }|                     |          5 }|                    d           d d d            n# 1 swxY w Y   | \  }}}	}
d|	 d|
 d| d| d| 
}t          |d	z            }t          ||           t	                      }|                    |          5 }|                                dk    sJ 	 d d d            n# 1 swxY w Y   t          |d
z            }t          |||           |                    |          5 }|                                dk    sJ 	 d d d            n# 1 swxY w Y   t          |dz            }t          |          }t          ||           |                    |          5 }|                                dk    sJ 	 d d d            n# 1 swxY w Y   t          |dz            }t          |||           |                    |          5 }|                                dk    sJ 	 d d d            n# 1 swxY w Y   t          |dz            }t          ||dd           |                    |          5 }|                                dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   r   rh  r0  rc  r   @z?scheme=http&endpoint_override=zc_copied1.txtzc_copied2.txt)source_filesystemzc_copied3.txtzc_copied4.txt)destination_filesystemzc_copied5.txtrz  F
chunk_sizeuse_threads)r}   r4  r   r   r   rt   r  r   )s3_connectionr   r   r   r   r4   r6  r   r   r   r   
source_urilocal_path1r   local_path2local_path3destination_urilocal_path4local_path5s                      r    test_copy_filesr    si   	dB(^F 6'??D			t	$	$ 	               *7&D$
J	8
 	8 	8Z 	8 	8$ 	8 	8*.	8 	815	8 	8 
 g/00Kz;'''G		"	";	/	/ #1vvxx7"""""# # # # # # # # # # # # # # # g/00Kt[B7777		"	";	/	/ #1vvxx7"""""# # # # # # # # # # # # # # # g/00K%k22Oz?+++		"	";	/	/ #1vvxx7"""""# # # # # # # # # # # # # # # g/00Kz;wGGGG		"	";	/	/ #1vvxx7"""""# # # # # # # # # # # # # # # g/00Kz;1%HHHH		"	";	/	/ #1vvxx7"""""# # # # # # # # # # # # # # # # # #sk   AAA;C##C'*C''EEE GGGH44H8;H89J""J&)J&c                    t                      | dz  }|                                                     t          |dz                      5 }|                    d           d d d            n# 1 swxY w Y                       t          |dz                      5 }|                    d           d d d            n# 1 swxY w Y   fd}| dz  }|                                 t          t          |          t          |                      ||           | dz  }|                                 t          t          |          t          |          	            ||           | d
z  }|                                 t          t          |                    }t          t          |                    }t          ||            ||           | dz  }	|	                                 t          ||	            ||	           | dz  }
|
                                 t          ||
dd            ||
           d S )Nsourcefile1   test1file2   test2c                 j                        t          | dz                      5 }|                                dk    sJ 	 d d d            n# 1 swxY w Y                        t          | dz                      5 }|                                dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr  r  r  r  )rt   r   r  )destination_dirr6  r   s     r    check_copied_filesz5test_copy_files_directory.<locals>.check_copied_files  s0   &&s?W+D'E'EFF 	(!6688x'''''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(&&s?W+D'E'EFF 	(!6688x'''''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(s#   AAA?B((B,/B,destination1destination2)r  r  destination3destination4destination5rz  Fr  )r   mkdirr}   r   r4  r   r   )r   
source_dirr6  r  destination_dir1destination_dir2destination_dir3r  r  destination_dir4destination_dir5r   s              @r    test_copy_files_directoryr    s   G 8#J		#	#C
W(<$=$=	>	> !	              		#	#C
W(<$=$=	>	> !	              ( ( ( ( ( /s:$4 5 5666'((( /s:$4 5 5!(J J J J'((( / Z11J%c*:&;&;<<Oz?+++'((( /z+,,,'((( /z+uMMMM'(((((s$   A00A47A4 CC	Cc                  L    d} t          j        t          j        d| g           d S )Na  if 1:
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)

        fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        assert fs.region == 'eu-west-1'
        f = fs.open_input_stream(path)
        f.read(50)

        finalize_s3()

        with pytest.raises(ValueError, match="S3 .* finalized"):
            f.read(50)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            fs.open_input_stream(path)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            S3FileSystem(anonymous=True)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        -c
subprocess
check_callsys
executablecodes    r    test_s3_finalizer    s+    D* 3>4677777r"   c                  L    d} t          j        t          j        d| g           d S )Na  if 1:
        import pytest
        from pyarrow.fs import resolve_s3_region, ensure_s3_initialized, finalize_s3

        resolve_s3_region('mf-nwp-models')

        finalize_s3()

        # Testing both cached and uncached accesses
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('mf-nwp-models')
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('arrow-datasets')
        r  r  r  s    r     test_s3_finalize_region_resolverr  ;  s+    D 3>4677777r"   c                  L    d} t          j        t          j        d| g           d S )Na  if 1:
        import threading
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)
        threads = []
        fn = lambda: FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        for i in range(4):
            thread = threading.Thread(target = fn)
            threads.append(thread)
            thread.start()

        for thread in threads:
            thread.join()

        finalize_s3()
        r  r  r  s    r    test_concurrent_s3fs_initr  O  s+    
D" 3>4677777r"   zUatexit(ensure_s3_finalized) will be called too late with bundled aws-sdk-cpp 1.11.587r"  z?Leaking S3ClientFinalizer causes segfault on musl based systemsc            	      b   	 t          j        ddg           n$# t          $ r t          j        d           Y nw xY wt                      } dddd|  dt          j                            t          d          g}t          j
        |t           j        	          }	 d
|  d}t          j                    }d }t          j                    |dz   k     r~	 t          |          5 }|j        dk    sJ 	 d d d            n# 1 swxY w Y   na# t          $ r }|}t          j        d           Y d }~nd }~ww xY wt          j                    |dz   k     ~t          j        d|d|            |                                 n# |                                 w xY w|                                dk    sJ d S )Nuwsgiz	--versionz"uwsgi not installed on this Pythonz-iz--httpz
127.0.0.1:z--wsgi-filezwsgi_examples.py)stdinzhttp://127.0.0.1:z/s3/r     g?zCould not fetch r      )r  r  rM   r   r   r   r  r4   joinherePopenDEVNULLtimer   statusr   sleepfail	terminatewait)r   argsprocurl
start_timeerrorrespr   s           r    test_uwsgi_integrationr  h  s   :w45555 : : :899999: DT8%8$%8%827<<.@AACDD
(:;;;D,$,,,Y[[
ikkJN** S\\ .T;#-----. . . . . . . . . . . . . . .      
3  ikkJN** K;3;;E;;<<<99;;"sf    ::5E< D C:.D :C>>D C>D E< 
D0D+&E< +D007E< <Fc                     	 ddl m}  ddlm} n$# t          $ r t          j        d           Y nw xY wt          j        d          \  }}t          t           |                                }||k    sJ |dk    sJ d}t          j        d|           \  }}t          t           |                                 }||k    sJ d S )	Nr   )r   MemoryFileSystemfsspec not installedz$fsspec+memory://path/to/data.parquetz/path/to/data.parquetzfile:///tmp/my.filezfsspec+)fsspec.implementations.localr   fsspec.implementations.memoryr  ImportErrorr   r   r   rH  r   r   )r   r  r   r4   expected_fsrJ  r5  s          r    test_fsspec_filesystem_from_urir    s   ,@@@@@@BBBBBBB , , ,*+++++, "#IJJHB}-=-=-?-?@@AAK*****  C#00EB}__->->??@@Ks    00c                  p   	 ddl m}  n$# t          $ r t          j        d           Y nw xY wt           |                                           dd                               dd                               di 	          5 }|                    d
           d d d            n# 1 swxY w Y                       di 	          5 }|                    d           d d d            n# 1 swxY w Y   fd} |d          t          j
        k    sJ  |d          t          j        k    sJ  |d          t          j
        k    sJ                                   |d          t          j        k    sJ  |d          t          j        k    sJ  |d          t          j        k    sJ d S )Nr   r  r  test_dirTr  ztest_dir/subdirztest_file.txtr  s   test contentztest_dir/nested_file.txts   nested contentc                 F                         | g          d         j        S )Nr   )rC   r<  )r4   r   s    r    get_typez6test_fsspec_delete_root_dir_contents.<locals>.get_type  s!    ''*//r"   )r  r  r  r   r   r   rT   r}   r4  r   r<   r=   r^   r>   )r  r  r  r   s      @r    $test_fsspec_delete_root_dir_contentsr    s   ,BBBBBBB , , ,*+++++, 
''))	*	*B MM*M---MM#tM444					<	< &_%%%& & & & & & & & & & & & & & & 
		9B		G	G (6&'''( ( ( ( ( ( ( ( ( ( ( ( ( ( (0 0 0 0 0 8O$$55558J8#555558.//8=@@@@ !!! 8O$$(999998J8#444448.//83DDDDDDDs0   
 ++
B,,B03B0C00C47C4c                     t          j        d           	 ddlm}  n$# t          $ r t          j        d           Y nw xY wt          j        d          \  }}t          t           |                                 }||k    sJ |dk    sJ d S )Nr   r   )HfFileSystemzhuggingface_hub not installedzFhf://datasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquetzAdatasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquet)
r   r   huggingface_hubr  r  r   r   rH  r   r   )r  r   r4   r  s       r    $test_huggingface_filesystem_from_urir    s    
!!!50000000 5 5 53444445 "P HB }\\^^<<==KVVVVVVVs    >>)r   r   r   r  r  r}  urllib.requestr   r  r  r  r   ru  r   rp   pyarrow.tests.test_ior   pyarrow.tests.utilr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   pyarrow.utilr   r4   dirnameabspathr  r  r   fixturer   r   r   r   r   r   r   r   r   r1  r   r   r   r   r  r	  markr   r   r   r
  r   r   r   r   r  r  r  r'  r*  r7  rC  rF  rO  rQ  rX  rd  rp  rs  ru  r  r  r  r  r  r  r  r  r  r  r  r  r  r  parametrizecompressr  r  r  
decompressr  r  r  r  r  r  r  r  r  r%  r,  r:  rV  r[  r]  ra  re  rh  rl  rq  rs  rx  rz  r|  r  r  r  r  r  r  r  r  r  r  r  r  	processes	threadingr  r   skipifr  r  r  r  r/   r"   r    <module>r     s  $ 3 2 2 2 2 2 2 2 2 2  				  " " " " " "     



        7 7 7 7 7 76 6 6 6 6 6 6 6 6 6 6 6$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ ( ' ' ' ' ' 
wrwx0011f' f' f' f' f'$ f' f' f'R                     :   4    @   8   $         , FL   FL+   FL1   FLkn  
 FLko  
 FLk  
 FLk  
 FL   FL:   FL:   FLB   FLE   FL=kn  k: : : :v2 2w: :v2 # # # ' ' ' / / / 5 5 5	2 	2 	2& & &
  > > >
# # #
  5  5  5F  :  
O O O(  K K K   ( ( (," " "- - -     8+ 8+ 8+v5  5  5 p  "  . . .@  $  .( ( (
	 	 	  (  (  $   0	tX	r8	t}%	dm$	 
 
  
& & &&      2	tX	r8	t'	do&	 	) 	)  	) @	tXx(	r8X&	t}do6	dmT_5	 ; ;  ;0" " ".# # #. . .(     "B "B "BJ d7 d7 d7N I3 I3 I3X ' ' '. Z Z Zz D1 D1 D1N C or"_i0oy13!?NC$#&oz24!?OD/3*-*<=!F  $! !% $!  " 
iX. 5 5	 5 + + +$ + + +&    (	- 	- 	-  *" " "., , ,:3 3 3() ) )' ' '     F F F.   B ' ' ' 1# 1# 1#h0) 0) 0)f 8 8 86 8 8 8& 8 8   8,  6 7 7((** 4T U U U U7 7 
@  ( E  E  EFW W W W Wr"   