
    PiH                        d dl mZ d dlmZmZ d dlmZ d dlZd dl	Z	d dl
Zd dl
mZmZ d dlmZ d dlmZ e	j                            d          Ze	j        d             Z e	j        d	d
g          d             Zd Zd Zd Zd Zd Zd Ze	j                            dddg          e	j                            deeg          d                         Z d Z!d Z"d Z#d Z$e	j                            dddg          d             Z%e	j                            dg d          d             Z&e	j                            dg d          d              Z'd! Z(e	j                            d"dd#g          d$             Z)e	j                            d%d&d'g          d(             Z*d) Z+d* Z,d+ Z-e	j                            d,g d-          d.             Z.d/ Z/e	j                            d0d1d2g          d3             Z0d4 Z1d5 Z2d6 Z3d7 Z4dS )8    )Iterator)BytesIOStringIO)PathN)	DataFrame	read_json)
JsonReaderz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningc                  \    t          g dg dd          } |                     dd          S )N                  ABTrecordslinesorient)r   to_json)dfs    w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/pandas/tests/io/json/test_readlines.pylines_json_dfr      s5    	33	4	4B::D:333    ujsonpyarrow)paramsc                 N    | j         dk    rt          j        d           | j         S )Nr    zpyarrow.json)parampytestimportorskip)requests    r   enginer'      s(    }	!!N+++=r   c                      t          t          d          d          } t          ddgddggddg          }t          j        | |           d S )	Nz!{"a": 1, "b": 2}
{"b":2, "a" :1}
Tr   r   r   abcolumns)r   r   r   tmassert_frame_equal)resultexpecteds     r   test_read_jsonlr2   &   sY    x EFFdSSSF1a&1a&)C:>>>H&(+++++r   c                     t           | dddd          d|          }t          g dg dd	          }t          j        ||           d S )
Niojsondataline_delimited.jsonTr   r'   )r   r   r   )r   r   r   r*   r+   )r   r   r.   r/   )datapathr'   r0   r1   s       r   test_read_jsonl_engine_pyarrowr;   -   sh    vv'<==  F
 yyyyyy99::H&(+++++r   c                    |dk    r;d}|                      t          j                            |t                               t          ddgddgdfgg d	
          }|                    dd          }|dk    rt          t          |          |          }nt          t          |          |          }t          g dg dgg d	
          }t          j
        ||           d S )Nr    zEPyarrow only supports a file path as an input and line delimited jsonreasonraisesr   r   
2020-03-052020-04-08T09:58:49+00:00hector)accountsdatenamer,   Tr   r   r'   )r   r@   rB   )r   rA   rB   )applymarkerr$   markxfail
ValueErrorr   r   r   r   r.   r/   )r&   r'   r>   r   	json_liner0   r1   s          r   test_read_datetimerL   7   s   XFK--VJ-OOPPP	a&<!<=x	HI,,,
 
 
B 

i
88I8I..v>>>8I..v>>>	$	$	$&P&P&PQ,,,  H &(+++++r   c                  B   d} t          |           } t          | d          }t          ddgddggddg	          }t          j        ||           d} t          t          |           d          }t          ddgddggddg	          }t          j        ||           d S )
Nu5   {"a": "foo”", "b": "bar"}
{"a": "foo", "b": "bar"}
Tr)   u   foo”barfoor*   r+   r,   )r   r   r   r.   r/   )r5   r0   r1   s      r   test_read_jsonl_unicode_charsrP   O   s    
 EDD>>Dt4(((F;.?#sTTTH&(+++ EDx~~T222F;.?#sTTTH&(+++++r   c                  &   t          ddgddggddg          } |                     dd          }d	}||k    sJ t          d
dgddggddg          } |                     dd          }d}||k    sJ t          j        t	          t          |          d          |            t          ddgddggddg          } |                     dd          }d}||k    sJ t          j        t	          t          |          d          |            d S )Nr   r   r*   r+   r,   r   Tr   r   z{"a":1,"b":2}
{"a":1,"b":2}
zfoo}rN   zfoo"z/{"a":"foo}","b":"bar"}
{"a":"foo\"","b":"bar"}
r)   zfoo\za\z4{"a\\":"foo\\","b":"bar"}
{"a\\":"foo\"","b":"bar"}
)r   r   r.   r/   r   r   )r   r0   r1   s      r   test_to_jsonlrS   a   sE   	QFQF#c3Z	8	8	8BZZyZ55F/HX	VUOfe_5Sz	J	J	JBZZyZ55FCHX)HV$4$4DAAA2FFF 
We$vuo6	M	M	MBZZyZ55FNHX)HV$4$4DAAA2FFFFFr   c                      t          ddgddggddg          } |                     dd                              d	          }d}||k    sJ d S )
Nr   r   r*   r+   r,   r   TrR   
)r   r   count)r   actual_new_lines_countexpected_new_lines_counts      r   test_to_jsonl_count_new_linesrY   v   sg    	QFQF#c3Z	8	8	8BZZyZEEKKDQQ !%=======r   	chunksizer   g      ?bufferc                    |t           k    r|                                }|dk    r;d}|                     t          j                            |t                               t           ||          d          } ||          5 }t          |d||          5 }t          j	        |          }	d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          j        |	|           d S )Nr    mPyarrow only supports a file path as an input and line delimited jsonand doesn't support chunksize parameter.r=   Tr)   r   rZ   r'   )r   encoderG   r$   rH   rI   rJ   r   pdconcatr.   r/   )
r&   r   rZ   r[   r'   r>   	unchunkedbufreaderchunkeds
             r   test_readjson_chunksrf   ~   ss    %,,..7 	 	FK--VJ-OOPPP&&//t<<<I}$!$#TYvFFF$JP)F##	$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ '9-----s6   CB;/C;B?	?CB?	CCCc                     d}t          j        t          |          5  t          t	          |           dd|          5 }	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz*chunksize can only be passed if lines=TruematchFr   r^   r$   r?   rJ   r   r   )r   r'   msg_s       r   &test_readjson_chunksize_requires_linesrm      s    
6C	z	-	-	-  ]##5Af
 
 
 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 4   !A%AA%A	A%A	A%%A),A)c                 
   |dk    r5d}|                      t          j                            |                     t	          j        ddd          }t          |                    dd	                    }t          |dd
|          }t          |                    dd	                    }t          |dd
d|          5 }t	          j	        |          }d d d            n# 1 swxY w Y   t          j        ||           d S )Nr    r]   )r>   r   r   r   Tr   r   series)r   typr'   )r   rq   rZ   r'   )rG   r$   rH   rI   r`   Seriesr   r   r   ra   r.   assert_series_equal)r&   r'   r>   sstriorb   rd   re   s           r   test_readjson_chunks_seriesrv      sF   7 	 	FK--V-<<=== 		""##AQYYT)Y<<==E%t&IIIIQYYT)Y<<==E	Tx1V
 
 
 $	)F##$ $ $ $ $ $ $ $ $ $ $ $ $ $ $
 7I.....s   C##C'*C'c                 b   |dk    r;d}|                      t          j                            |t                               t          t          |          dd|          5 }t          |          }d d d            n# 1 swxY w Y   |d         j        dk    sJ |d	         j        d
k    sJ d S )Nr    r]   r=   Tr   r^   r   r   r   r   )r   r   )	rG   r$   rH   rI   rJ   r   r   listshape)r&   r   r'   r>   rd   chunkss         r   test_readjson_each_chunkr|      s    7 	 	FK--VJ-OOPPP 
tq
 
 
 	f               !9?f$$$$!9?f$$$$$$s   "A>>BBc                    |dk    r;d}|                      t          j                            |t                               t          g dg dd          }|                    |dd	           t          |dd
|          5 }t          j	        |          }d d d            n# 1 swxY w Y   t          |d|          }t          j        ||           d S )Nr    r]   r=   r   r   r   Tr   r   r   r^   r8   )rG   r$   rH   rI   rJ   r   r   r   r`   ra   r.   r/   )r&   r'   	temp_filer>   r   rd   re   rb   s           r   test_readjson_chunks_from_filer      s   7 	 	FK--VJ-OOPPP	33	4	4BJJyYJ777	9DAf	E	E	E $)F##$ $ $ $ $ $ $ $ $ $ $ $ $ $ $)4???I)W-----s   B$$B(+B(c                 4   t          g dg dd          }|                    |dd           t          |d ddddddd d d| d d 	          }|5  |                                 d d d            n# 1 swxY w Y   |j        j        j        sJ d
|              d S )Nr   r   r   Tr   r   frameF)r   rq   dtypeconvert_axesconvert_dateskeep_default_datesprecise_float	date_unitencodingr   rZ   compressionnrowsz%didn't close stream with chunksize = )r   r   r	   readhandleshandleclosed)rZ   r~   r   rd   s       r   test_readjson_chunks_closesr      s   	33	4	4BJJyYJ777  F  
                > '  ;	;; '  s   A//A36A3)r   g@rO   c                     d}t          j        t          |          5  t          t	          |           d||          5 }	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz"'chunksize' must be an integer >=1rh   Tr^   rj   )r   rZ   r'   rk   rl   s        r   test_readjson_invalid_chunksizer      s    
/C	z	-	-	-  ]##49V
 
 
 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 rn   )Nr   r   c                    d}t          g dg dd          }t          t          |          d|           }| .|5  t          j        |          }d d d            n# 1 swxY w Y   t          j        ||d|             d S )	NzF

    {"A":1,"B":4}



    {"A":2,"B":5}







    {"A":3,"B":6}
    r   r   r   Tr   rZ   zchunksize: )obj)r   r   r   r`   ra   r.   r/   )rZ   jorigtests       r   )test_readjson_chunks_multiple_empty_linesr     s    	A  9999995566DXa[[	BBBD 	# 	#9T??D	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	#$*C	*C*CDDDDDDs   AA!$A!c                    |dk    r;d}|                      t          j                            |t                               |                    dd            t          |dd          5 }|                    d	           d d d            n# 1 swxY w Y   t          ||
          }t          ddgi          }t          j        ||           d S )Nr    r]   r=   zlocale.getpreferredencodingc                     dS )Ncp949 )do_setlocales    r   <lambda>z'test_readjson_unicode.<locals>.<lambda>)  s    G r   wzutf-8)r   u6   {"£©µÀÆÖÞßéöÿ":["АБВГДабвгд가"]}rF   u   £©µÀÆÖÞßéöÿu   АБВГДабвгд가)rG   r$   rH   rI   rJ   setattropenwriter   r   r.   r/   )r&   monkeypatchr'   r~   r>   fr0   r1   s           r   test_readjson_unicoder      s+   7 	 	FK--VJ-OOPPP57S7STTT	iw	/	/	/ J1	HIIIJ J J J J J J J J J J J J J J y000F25N4OPQQH&(+++++s   +BBBr   r   c                     d}t          t          |          d|           }t          g dg dd          j        d |          }t	          j        ||           d S )N[{"a": 1, "b": 2}
        {"a": 3, "b": 4}
        {"a": 5, "b": 6}
        {"a": 7, "b": 8}T)r   r   r   r   r      r   r   r      r9   )r   r   r   ilocr.   r/   )r   r'   jsonlr0   r1   s        r   test_readjson_nrowsr   2  sg    E xd%@@@F|||,,,??@@EfufMH&(+++++r   znrows,chunksizerx   )r   r   c                 $   |dk    r;d}|                      t          j                            |t                               d}|dk    rNt          t          |          d|||          5 }t          j        |          }d d d            n# 1 swxY w Y   n@t          |d|||          5 }t          j        |          }d d d            n# 1 swxY w Y   t          g dg dd	          j
        d |         }t          j        ||           d S )
Nr    r]   r=   r   T)r   r   rZ   r'   r   r   r9   )rG   r$   rH   rI   rJ   r   r   r`   ra   r   r   r.   r/   )	r&   r   rZ   r'   r>   r   rd   re   r1   s	            r   test_readjson_nrows_chunksr   ?  s    7 	 	FK--VJ-OOPPPE
 UOO4u	RX
 
 
 	(i''G	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(
 Ui
 
 
 	(i''G	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( |||,,,??@@EfufMH'8,,,,,s$   +BBB,CCCc                     d}d}t          j        t          |          5  t          |dd|            d d d            d S # 1 swxY w Y   d S )Nr   z&nrows can only be passed if lines=Truerh   Fr   )r   r   r'   )r$   r?   rJ   r   )r'   r   rk   s      r   "test_readjson_nrows_requires_linesr   ^  s    E 3C	z	-	-	- > >%uAf====> > > > > > > > > > > > > > > > > >s   AAAc                 .   |dk    r;d}|                      t          j                            |t                               t          ddggddgdg	          t          d
dggddgdg	          t          ddggddgdg	          g} |dddd          }t          |                                          }t          |dd|          5 }t          |          D ] \  }}	t          j        |	||                    !	 d d d            d S # 1 swxY w Y   d S )Nr    r]   r=   r   r   r*   r+   r   )r-   indexr   r   r   r   r4   r5   r6   r7   Tr^   )rG   r$   rH   rI   rJ   r   r   as_urir   	enumerater.   r/   )
r&   r:   r'   r>   df_list_expectedos_pathfile_url
url_readerr   chucks
             r   "test_readjson_lines_chunks_fileurlr   j  s    7 	 	FK--VJ-OOPPP 	Aq6(S#Jqc:::Aq6(S#Jqc:::Aq6(S#Jqc:::
 htVV-BCCGG}}##%%H	841V	D	D	D B
%j11 	B 	BLE5!%)9%)@AAAA	BB B B B B B B B B B B B B B B B B Bs   1D

DDc            	          ddz  }  G d d          } ||           }t          t          t          |dd                              dk    sJ |j        d	k    sJ d S )
Nz\{"a": 1, "b": 2}
        {"a": 3, "b": 4}
        {"a": 5, "b": 6}
        {"a": 7, "b": 8}
i  c                   (    e Zd ZddZd ZdefdZdS )/test_chunksize_is_incremental.<locals>.MyReaderreturnNc                 <    d| _         t          |          | _        d S )Nr   )
read_countr   stringio)selfcontentss     r   __init__z8test_chunksize_is_incremental.<locals>.MyReader.__init__  s    DO$X..DMMMr   c                 @    | xj         dz  c_          | j        j        | S Nr   )r   r   r   )r   argss     r   r   z4test_chunksize_is_incremental.<locals>.MyReader.read  s$    OOq OO%4=%t,,r   c                 J    | xj         dz  c_         t          | j                  S r   )r   iterr   )r   s    r   __iter__z8test_chunksize_is_incremental.<locals>.MyReader.__iter__  s"    OOq OO&&&r   )r   N)__name__
__module____qualname__r   r   r   r   r   r   r   MyReaderr     sR        	/ 	/ 	/ 	/	- 	- 	-	'h 	' 	' 	' 	' 	' 	'r   r   Td   r   r   
   )lenry   r   r   )r   r   rd   s      r   test_chunksize_is_incrementalr     s    	 		 
' ' ' ' ' ' ' ' Xe__FtIfDC@@@AABBQFFFFr!!!!!!r   orient_)splitr   tablec                     t          ddgddgd          }d}t          j        t          |          5  |                    d|            d d d            d S # 1 swxY w Y   d S )	Nr   r   r*   r+   col1col2Pmode='a' \(append\) is only supported when lines is True and orient is 'records'rh   )moder   r   r$   r?   rJ   r   )r   r   rk   s      r   test_to_json_append_orientr     s     
QFS#J77	8	8B	0  
z	-	-	- - -


G
,,,- - - - - - - - - - - - - - - - - -s   AA Ac                      t          ddgddgd          } d}t          j        t          |          5  |                     ddd	
           d d d            d S # 1 swxY w Y   d S )Nr   r   r*   r+   r   r   rh   Fr   r   r   r   r   )r   rk   s     r   test_to_json_append_linesr     s     
QFS#J77	8	8B	0  
z	-	-	- < <


5
;;;< < < < < < < < < < < < < < < < < <s   AA!Amode_rxc                     t          ddgddgd          }d|  d}t          j        t          |          5  |                    | d	d
           d d d            d S # 1 swxY w Y   d S )Nr   r   r*   r+   r   zmode=z@ is not a valid option.Only 'w' and 'a' are currently supported.rh   Fr   r   r   )r   r   rk   s      r   test_to_json_append_moder     s     
QFS#J77	8	8BWWWW  
z	-	-	- > >


U9
===> > > > > > > > > > > > > > > > > >s   AA"%A"c                 8   t          ddgddgd          }t          ddgdd	gd          }t          g d
g dd          }|                    | dd           |                    | ddd           t          | d          }t          j        ||           d S )Nr   r   r*   r+   r   r   r   cd)r   r   r   r   )r*   r+   r   r   Tr   r   r   r)   )r   r   r   r.   r/   )r~   df1df2r1   r0   s        r   -test_to_json_append_output_consistent_columnsr     s     aVc3Z88
9
9C
aVc3Z88
9
9C,,,8L8L8LMMNNHKK	iK888KK	4	KBBB y---F&(+++++r   c                 l   t          ddgddgd          }t          ddgdd	gd
          }t          g dg dt          j        t          j        dd	gd          }|                    | ddd           |                    | ddd           t	          | d          }t          j        ||           d S )Nr   r   r*   r+   r   er   !#r   col3)r   r   NN)r*   r+   r   r   )r   r   r   Tr   r   r)   )r   npnanr   r   r.   r/   )r~   r   df3r1   r0   s        r   /test_to_json_append_output_inconsistent_columnsr     s     aVc3Z88
9
9C
c3Z#s<<
=
=C&&&(((VRVS#.	
 	
 H KK	4	KBBBKK	4	KBBB y---F&(+++++r   c                    t          ddgddgd          }t          ddgdd	gd          }t          d
dgddgd          }t          dddgi          }t          g ddddd	d
dt          j        t          j        gt          j        t          j        t          j        t          j        ddt          j        t          j        gg dd                              ddi          }|                    | ddd           |                    | ddd           |                    | ddd           |                    | ddd           t          | d          }t          j        ||           d S )Nr   r   r*   r+   r   r   r   r   r   r   r   r   r   r   col4TF)r   r   r   r   NNNN)NNNNNNTF)r   r   r   r   floatr   r   r)   r   r   r   astyper   r   r.   r/   r~   r   r   r   df4r1   r0   s          r   ,test_to_json_append_output_different_columnsr     s    aVc3Z88
9
9C
aVc3Z88
9
9C
c3Z#s<<
=
=C
VdE]+
,
,C888#sCc2626BVRVRVRVS#rvrvNEEE		
 	
  ffg  KK	4	KBBBKK	4	KBBBKK	4	KBBBKK	4	KBBB y---F&(+++++r   c                    t          ddgddgd          }t          ddgdd	gd          }t          d
dgddgd          }t          dddgi          }t          g dt          j        t          j        d
ddd	ddgt          j        t          j        ddt          j        t          j        t          j        t          j        gg dd                              ddi          }|                    | ddd           |                    | ddd           |                    | ddd           |                    | ddd           t          | d          }t          j        ||           d S )Nr   r   r*   r+   r   r   r   r   r   r   r   r   r   r   r   TF)TFNNNNNN)NNNNr   r   r   r   )r   r   r   r   r   r   r   r)   r   r   s          r   6test_to_json_append_output_different_columns_reorderedr    s    aVc3Z88
9
9C
aVc3Z88
9
9C
c3Z#s<<
=
=C
VdE]+
,
,C EEEVRVS#sCcBVRVS#rvrvrvrvN888		
 	
  ffg  KK	4	KBBBKK	4	KBBBKK	4	KBBBKK	4	KBBB y---F&(+++++r   )5collections.abcr   r4   r   r   pathlibr   numpyr   r$   pandasr`   r   r   pandas._testing_testingr.   pandas.io.json._jsonr	   rH   filterwarnings
pytestmarkfixturer   r'   r2   r;   rL   rP   rS   rY   parametrizerf   rm   rv   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   <module>r     s   $ $ $ $ $ $                                    + + + + + +[''C 

 4 4 4
 +,,,  -,, , ,, , ,, , ,0, , ,$G G G*> > > q#h//GX#677. . 87 0/.6  / / /0% % %&. . ." tQi00  104 &9&9&9::  ;: lll33E E 43E2, , ,$ 1a&))	, 	, *)	, *VV,<==- - >=-<	> 	> 	>B B B." " "8 $?$?$?@@	- 	- A@	-	< 	< 	< 3*--> > .->, , ,", , ,., , ,8, , , , ,r   