&`i*ddlZddlZddlmZmZmZmZmZmZddl Z ddl m Z ddl mZddlmZddlmZerddlZejeZgdZGdd eZGd d eZd e jfd ZGddejZdS)N) TYPE_CHECKINGAnyDictListOptionalUnion)pyarrow_table_from_pydict)PandasBlockAccessor) DataContext)FileBasedDatasource) jsonjsonlzjson.gzzjsonl.gzzjson.brzjsonl.brzjson.zstz jsonl.zstzjson.lz4z jsonl.lz4ceZdZdZdddeeeefdeeee fffdZ dd Z dd Z d d d efdZ xZS)ArrowJSONDatasourcez>JSON datasource, for reading and writing JSON and JSONL files.N)arrow_json_argspathsrc ddlm}tj|fi||i}|d|d|_||_dS)Nr)r read_optionsF) use_threads)pyarrowr super__init__pop ReadOptionsrr)selfrrfile_based_datasource_kwargsr __class__s /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/data/_internal/datasource/json_datasource.pyrzArrowJSONDatasource.__init__&s !     ??">???  " O+// D,,,??   /bufferpyarrow.lib.Bufferc#>Kddl}ddlm}|jj}t jj} |jtj |fd|ji|j V||j_dS#|j $r}dt|vr||jj|krNtd|jjd|jjdzd |jxjdzc_n'| |d |jjd |Yd}~nd}~wwxYw) zRead with PyArrow JSON reader, trying to auto-increase the read block size in the case of the read object straddling block boundaries.rNTrz0straddling object straddles two block boundariesz+JSONDatasource read failed with block_size=z. Retrying with block_size=.z! - Auto-increasing block size to a+ bytes failed. Please try manually increasing the block size through the `read_options` parameter to a larger size. For example: `read_json(..., read_options=pyarrow.json.ReadOptions(block_size=10 << 25))`More information on this issue can be found here: https://github.com/apache/arrow/issues/25674)r pyarrow.jsonr r block_sizer get_currenttarget_max_block_size read_jsonioBytesIOr ArrowInvalidstrloggerdebug)rr papajsoninit_block_sizemax_block_sizees r_read_with_pyarrow_read_jsonz0ArrowJSONDatasource._read_with_pyarrow_read_json9s %%%%%%$+6$022H# " &f&Jv&&!%!2* 0?!,?   EQOO&.,7.HH N*.*;*FNN*.*;*F*JNNN )449444 oo LL#0;LLL   G54444 # s:A// D9BDDc#Kddl}ddl}|jdkrdS|t j|} |j|VdS#t$r}dt|vsJt|ddl m }|t}|D]7}| D] \} } || | !8t|VYd}~dSd}~wwxYw)z{Fallback method to read JSON files with Python's native json.load(), in case the default pyarrow json reader fails.rNzno attribute 'from_pylist') defaultdict)r rsizeloadr*r+Table from_pylistAttributeErrorr- collectionsr7listitemsappendr ) rr r r0 parsed_jsonr4r7dctrowkvs r_read_with_python_jsonz*ArrowJSONDatasource._read_with_python_jsonwsH   ;!   Fii 6 2 233  1(&&{33 3 3 3 3 3 1 1 103q669993q66999 / / / / / /+d##C" % %IIKK%%DAqFMM!$$$$%+C00 0 0 0 0 0 0 0 0 0 1sA C<(B C77C<fpyarrow.NativeFilepathc#Kddl}|} ||Ed{VdS#|j$rC}td|||Ed{VYd}~dSd}~wwxYw)NrzyError reading with pyarrow.json.read_json(). Falling back to native json.load(), which may be slower. PyArrow error was: )r read_bufferr5r,r.warningrF)rrGrIr0r r4s r _read_streamz ArrowJSONDatasource._read_streams !  ;88@@ @ @ @ @ @ @ @ @ @ ; ; ; NN+'(++    226:: : : : : : : : : : : : : : ;s9 B8BB)r r!)__name__ __module__ __qualname____doc__rr-rrrrrr5rFrM __classcell__rs@rrr#sHH 59 ///S$s)^$/"$sCx.1 //////&<<<<|11118;2;#;;;;;;;;rrceZdZdZdZdeeeefdeffd Z dddefd Z ddd e efd Z d d ded dffd Z xZS)PandasJSONDatasourceii'rtarget_output_size_bytesc Jtj|fi|||_dSN)rr_target_output_size_bytes)rrrVrrs rrzPandasJSONDatasource.__init__s3 ??">???)A&&&rrGrHrIc#FK||}t||j}|*tj||d}t |VdStj||d5}|D]}t |V ddddS#1swxYwYdS)N buffer_sizeT chunksizelines)_estimate_chunksizeStrictBufferedReader _BUFFER_SIZEpdr)_cast_range_index_to_string)rrGrIr^streamdfreaders rrMz!PandasJSONDatasource._read_streams ,,Q// %aT5FGGG  f FFFB-b11 1 1 1 1 1f FFF :& ::B5b999999: : : : : : : : : : : : : : : : : : :s1BBBreturnc|s|jS|dks Jd|jdSt ||j}t j|dd5} tt|}n#t$rYddddSwxYw dddn #1swxYwYtj |}| dkrd}nN|| z }tt!|j|z d}|d|S)z{Estimate the chunksize by sampling the first row. This is necessary to avoid OOMs while reading the file. rz%File pointer must be at the beginningNr[Tr])seekable_DEFAULT_CHUNK_SIZEtellrYrarbrcr)rdnext StopIterationr for_blocknum_rows size_bytesmaxroundseek)rrGrergrfblock_accessorr^ bytes_per_rows rr`z(PandasJSONDatasource._estimate_chunksizes zz|| ,+ +vvxx1}}}E}}}  ) 14%aT5FGGG \&AT : : : f 0f>>                              -6r::  " " $ $ ) )II*5577.:Q:Q:S:SSME$"@="PQQSTUUI q s64B<6BB< B,B<+B,,B<<CC filesystemzpyarrow.fs.FileSystemc |||}|||Stj||fi|SrX)resolve_compressionopen_input_filer_open_input_source)rrxrI open_args compressionrs rr|z'PandasJSONDatasource._open_input_sourcesU..tY??  --d33 3)uww)*dHHiHHHr)rNrOrPrbrlrr-rintrrMrr`r|rRrSs@rrUrUsL BS$s)^$B#&BBBBBB :2 :# : : : :%9hsm@ I+ I I  I I I I I I I I I IrrUrfct|jtjr$|jt |_|SrX) isinstancecolumnsrc RangeIndexastyper-)rfs rrdrds6"*bm,,,Z&&s++ IrcPeZdZdZdejdefdZd dZde fdZ fd Z xZ S) raaWrapper that prevents premature file closure and ensures full-buffered reads. This is necessary for two reasons: 1. The datasource reads the file twice -- first to sample and determine the chunk size, and again to load the actual data. Since pandas assumes ownership of the file and may close it, we prevent that by explicitly detaching the underlying file before closing the buffer. 2. pandas wraps the file in a TextIOWrapper to decode bytes into text. TextIOWrapper prefers calling read1(), which doesn't prefetch for random-access files (e.g., from PyArrow). This wrapper forces all reads through the full buffer to avoid inefficient small-range S3 GETs. filer\c<tj|||_dS)Nr[)r*BufferedReader_file)rrr\s rrzStrictBufferedReader.__init__s&tEEE rc6|j|SrX)rread)rr8s rrzStrictBufferedReader.readszt$$$rrhcdS)NT)rs rreadablezStrictBufferedReader.readablestrc|jsT|j|jt dSdSrX)closedrdetachcloser)rrs rrzStrictBufferedReader.closesT{  J      J      GGMMOOOOO  r)r) rNrOrPrQr* RawIOBaserrrboolrrrRrSs@rraras  FR\FFFFF%%%%$rra)r*loggingtypingrrrrrrpandasrc$ray.air.util.tensor_extensions.arrowr ray.data._internal.pandas_blockr ray.data.contextr )ray.data.datasource.file_based_datasourcer r getLoggerrNr.JSON_FILE_EXTENSIONSrrU DataFramerdrrarrrrs BBBBBBBBBBBBBBBBJJJJJJ??????((((((IIIIIINNN  8 $ $$~;~;~;~;~;-~;~;~;BRIRIRIRIRI.RIRIRIjBL2<r