
    Pi(                        d Z ddlmZmZ ddlZddlZddlZddlZddl	m
Z
mZ ddlmZ ej                            d          Zej                            d          Zd Zed             Zeej                            d	d
dg          ej                            dg d          d                                     Zd Zd Zej                            ddi dgfdddidgfdddgiddgfddgdddgfddgddej        dgfg          d              Zd! Zej                            d"g d#          d$             Zej                            d%ddg          d&             Zd' Zej                            dg d(          d)             Z ej                            dg d*          d+             Z!d, Z"d- Z#ej                            d.          ej                            d/d0d1g          d2                         Z$dS )3zZ
Tests encoding functionality during parsing
for all of the parsers defined in parsers.py
    )BytesIOTextIOWrapperN)	DataFrameread_csvz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningpyarrow_skipc                     d}| }t          d                    |                    }|                    |d|          }t          ddggddg	          }t	          j        ||           d S )
Ncp1255u   שלום:1234
562:123:)sepencodingi2  {   u   שלום1234columnsr   encoder   r   tmassert_frame_equal)all_parsersr   parserdataresultexpecteds         x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/pandas/tests/io/parser/test_encoding.pytest_bytes_io_inputr      su    HF+228<<==D__TsX_>>F3*
F/CDDDH&(+++++    c                     | }t          d                                          }|                    |ddd           }t          ddgg          }t	          j        ||           d S )Nu   Łaski, Jan;1;utf-8)r   r   headeru   Łaski, Jan   r   )r   r   r   r   r   s        r   test_read_csv_unicoder"   (   si    F&--//00D__TsWT_JJF,a0122H&(+++++r   r   ,	r   )utf-16zutf-16lezutf-16bec                    | }d                     d|          }|dd}d}|                    |          }t          |d          5 }	|	                    |           d d d            n# 1 swxY w Y   t	          t          |                    |                    |          5 }
 |j        |fd|i|} |j        |
fd|i|}d d d            n# 1 swxY w Y   t          j        ||           d S )	Nz)skip this
skip this too
A,B,C
1,2,3
4,5,6r#      )r   skiprowsr   wbr   r   )	replacer   openwriter   r   r   r   r   )r   r   r   	temp_filer   r   kwargsutf8
bytes_datafbytes_bufferr   r   s                r   test_utf16_bom_skiprowsr4   2   s   
 F	 
c			 	
 a((FDX&&J	i		 !	
               
wt{{40011D	A	A	A J\ HHXHHH"6?<II$I&IIJ J J J J J J J J J J J J J J &(+++++s$   A''A+.A+#!CCCc                     t           j                            |d          }| }|                    |dd          }t	          |          dk    sJ d S )Nzutf16_ex.txtr%   r$   )r   r   2   )ospathjoinr   len)r   csv_dir_pathr8   r   r   s        r   test_utf16_exampler<   K   sP    7<<n55DF__TH$_??Fv;;"r   c                     t           j                            |d          }| }|                    |d d          }|                    d          }|d         d         }d}||k    sJ d S )Nunicode_series.csvlatin-1)r    r   r   r!   i`  u$   Á köldum klaka (Cold Fever) (1994))r7   r8   r9   r   	set_index)r   r;   r8   r   r   gotr   s          r   test_unicode_encodingrB   R   sj    7<<&:;;DF__T$_CCFa  F
)D/C9H(??????r   zdata,kwargs,expectedza
1r!   z"a"
1	quotechar"zb
1namesab1
1T)rE   skip_blank_linesFc                    | }ddfd}|j         dk    r1|dk    r+|                    dd          rt          j        d	            |j         ||          fd
i|}t          d|i          }t          j        ||           d S )Nu   ﻿r   c                 R    | z                                  }t          |          S )N)r   r   )_databom_databomr0   s     r   _encode_data_with_bomz,test_utf8_bom.<locals>._encode_data_with_bomw   s(    %K''--x   r   pyarrowrI   rJ   Tz,https://github.com/apache/arrow/issues/38676)reasonr   rF   )enginegetpytestskipr   r   r   r   )	r   r   r/   r   r   rP   r   rO   r0   s	          @@r   test_utf8_bomrW   ^   s    * F
CD! ! ! ! ! !
 	""EMMJJ)400  	IJJJJV_22488RR4R6RRF#x))H&(+++++r   c                     t          dgdgd          }| }|                    |          }d                    |          }|                    t	          |          |          }t          j        ||           d S )Ng333333@test)mb_num	multibytezmb_num,multibyte
4.8,testr*   )r   formatr   r   r   r   r   )r   	utf_valueencoding_fmtr   r   r   r   r   s           r   test_read_csv_utf_aliasesr_      sz    SEAABBHF""9--H'..x88D__WT]]X_>>F&(+++++r   zfile_path,encoding)))ior   csvz	test1.csvr   ))r`   r   r   r>   r?   ))r`   r   r   zsauron.SHIFT_JIS.csvshiftjisc                 z   | } || }|                     ||          }t          ||          5 }|                     |          }|j        rJ 	 d d d            n# 1 swxY w Y   t          j        ||           t          |d          5 }	|                     |	|          }|	j        rJ 	 d d d            n# 1 swxY w Y   t          j        ||           t          |dd          5 }	|                     |	|          }|	j        rJ 	 d d d            n# 1 swxY w Y   t          j        ||           d S )Nr*   rb)moder   )re   	buffering)r   r,   closedr   r   )
r   	file_pathr   datapathr   fpathr   far   fbs
             r   test_binary_mode_file_buffersrm      s    FHi Eux88H	eh	'	'	' 2$$9}               (F+++	e$			 2h779}               (F+++	e$!	,	,	, h779}               (F+++++s5   AA #A !B;;B?B?-!DD"Dpass_encodingc                    | }|                     |          }|j        dk    r|du r|dv rt          j        d           t	          ddgi          }|                    d|          5 }|                    d	           |                    d
           |                    ||r|nd           }	t          j
        |	|           d d d            d S # 1 swxY w Y   d S )NrQ   T)       zThese cases freezefoobarzw+re   r   zfoo
barr   r*   )r\   rS   rU   rV   r   r,   r-   seekr   r   r   )
r   r]   r^   rn   r.   r   r   r   r2   r   s
             r   test_encoding_temp_filerv      s+   
 F""9--H}	!!mt&;&;	X@U@U()))%%)**H	TH	5	5 0	
	q			-PXXDQQ
fh///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   (ACCCc                    | }d}d}d}t          ||gi          }t          j                    5 }|                    | d|                     |                     |                    d           |                    ||          }t          j        ||           |j	        rJ 	 d d d            d S # 1 swxY w Y   d S )Nz	shift-jisu	   てすとu   こむ
r   r*   )
r   tempfileNamedTemporaryFiler-   r   ru   r   r   r   rg   )r   r   r   titler   r   r2   r   s           r   test_encoding_named_temp_filer|      s   FHED%$))H		$	&	& !	5""D""))(33444	q			X66
fh///8|                 s   A8B44B8;B8)r   r%   z	utf-16-bez	utf-16-lezutf-32c                     d}t          |                    |                     }t          |d|           }t          ddgddgdd	ggd
dg          }t	          j        ||           d S )Nu   a	b
：foo	0
bar	1
baz	2r$   )	delimiterr   u   ：foor   rs   r!   bazr'   rF   rG   )r   r   r   )r   r   encoded_datar   r   s        r   %test_parse_encoded_special_charactersr      s     -D4;;x0011LldXFFFFmeQZ%4c
  H &(+++++r   )r   Nr%   r	   r?   c                 z   | }t          g dg dg dd          }|                    |d|           |j        dk    rOd}t          j        t
          |	          5  |                    ||d
           d d d            n# 1 swxY w Y   d S |                    ||d
          }t          j        ||           d S )N)Raphael	DonatellozMiguel AngelLeonardo)redpurpleorangeblue)saizbo staffnunchunkkatana)namemaskweaponF)indexr   rQ   BThe 'memory_map' option is not supported with the 'pyarrow' enginematchT)r   
memory_map)	r   to_csvrS   rU   raises
ValueErrorr   r   r   )r   r   r.   r   r   msgdfs          r   test_encoding_memory_mapr      s1    FHHH777???	
 	
 H OOIUXO>>>}	!!R]:S111 	K 	KOOITOJJJ	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K	X$	G	GB"h'''''s   BBBc                    | }t          dgdz            }d|j        d<   |                    |ddd           |j        d	k    rOd
}t	          j        t          |          5  |                    |dd           ddd           n# 1 swxY w Y   dS |                    |dd          }t          j	        ||           dS )zO
    Chunk splits a multibyte character with memory_map=True

    GH 43540
    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaai   )r   u   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaąi  Fr   r   r    r   rQ   r   r   NT)r    r   )
r   ilocr   rS   rU   r   r   r   r   r   )r   r.   r   r   r   dfrs         r    test_chunk_splits_multibyte_charr     s!    F	d*	+	+	+B %BGDMIIiuUWIEEE}	!!R]:S111 	E 	EOOIdtODDD	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E
//)DT/
B
BC#r"""""s   "BBBc           	         g }d}d}d}t          t          |          t          |          |          D ]q}d                    d t          ||dz             D                       dz   }	 |                    d           n# t          $ r Y Xw xY w|                    |           r| }t          |          }	|	                    |ddd	           |j        d
k    rPd}
t          j
        t          |
          5  |                    |ddd           ddd           n# 1 swxY w Y   dS |                    |ddd          }t          j        |	|           dS )zg
    GH 43787

    Test correct handling of UTF-8 chars when memory_map=True and encoding is UTF-8
        u   𐂀 c                 ,    g | ]}t          |          S  )chr).0cs     r   
<listcomp>z,test_readcsv_memmap_utf8.<locals>.<listcomp>,  s    AAA1AAAAr   rx   r   Fr   rQ   r   r   NT)r    r   r   )rangeordr9   r   UnicodeEncodeErrorappendr   r   rS   rU   r   r   r   r   r   )r   r.   linesline_length
start_charend_charlnumliner   r   r   r   s               r   test_readcsv_memmap_utf8r     s    EKJH c*oos8}}kBB  wwAAdD4K(@(@AAABBTI	KK    ! 	 	 	H	TF	5		BIIiuUWIEEE}	!!R]:S111 	W 	WOOIdtgOVVV	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W
//)DTG/
T
TC"c"""""s$   *B  
BB8DD"%D"pyarrow_xfailre   zw+bzw+tc                 B   | }d}d|v rd}t          j        |d          5 }|                    |           |                    d           |                    |          }d d d            n# 1 swxY w Y   t          g dg          }t          j        ||           d S )Ns   abcdtabcdr   rt   r   r   )ry   SpooledTemporaryFiler-   ru   r   r   r   r   )r   re   r   contenthandler   r   s          r   test_not_readabler   @  s     FG
d{{		&D7	C	C	C %vWA__V$$% % % % % % % % % % % % % % % fX...H"h'''''s   A A--A14A1)%__doc__r`   r   r   r7   ry   numpynprU   pandasr   r   pandas._testing_testingr   markfilterwarnings
pytestmarkusefixturesskip_pyarrowr   r"   parametrizer4   r<   rB   nanrW   r_   rm   rv   r|   r   r   r   r   r   r   r   r   <module>r      s   
        
			                   [''C 
 {&&~66, , , , , , d,,%G%G%GHH, , IH -, ,,  	 	 	  
aS	K%s+	'C5!C:.	3%T::QC@ e77VQK	
 &, ,' &,.	, 	, 	,    , , ,0 4-880 0 980*  ( GGG , , , %S%S%STT( ( UT(,# # #2# # #B ))%00( ( 10 *)( ( (r   