&`inddlZddlmZddlmZmZmZmZmZm Z m Z m Z m Z ddl ZddlmZddlmZddlmZer ddlZddlmZdddd ddd d ee d d ede ddede edede ede eeefdee dfdZGddeZGddeZdS)N)chain) TYPE_CHECKINGAnyDictIterableListLiteralOptionalTupleUnion) _check_import) BlockAccessor)Datasink)FragmentMetadatai)schemamax_rows_per_filemax_bytes_per_filemax_rows_per_groupdata_storage_versionstorage_optionsstream)zpa.Table pd.DataFrameurir pa.Schemarrrrrreturn)rrc ddl}ddlm} m} t } t | |jr2tj |  n| j tj dkrdt| gfd} || n|}tj| } | | ||||||}fd|DS)Nr)DEFAULT_MAX_BYTES_PER_FILEwrite_fragmentsc3KD]B}tj|}|Ed{VCdSN)r for_blockto_arrow to_batches)blocktblrs /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/data/_internal/datasource/lance_datasink.pyrecord_batch_converterz/_write_fragment..record_batch_converter4sa ( (E)%0099;;C~~'' ' ' ' ' ' ' ' ' ( ()rrrrrrcg|]}|fSr+.0fragmentrs r' z#_write_fragment..Hs 9 9 98Xv  9 9 9r))pandaslance.fragmentrrnext isinstance DataFramepaSchema from_pandasremove_metadatarlennamesrRecordBatchReader from_batches)rrrrrrrrpdrrfirstr(reader fragmentss` ` r'_write_fragmentrAs@JJJJJJJJ ~V  eR\ * * "Y**511AACCFF\F v|   ! !Fw''((((( '9&@""FX ! . .v7M7M7O7O P PF +--1'   I : 9 9 9y 9 9 99r)c eZdZdZ ddedeejdeddee ee fffd Z e d e fd Zdded d dfd ZdeeeeeffdZxZS)_BaseLanceDatasinkzBase class for Lance Datasink.NcreaterrmoderDappend overwriterctj|i|||_||_||_d|_||_dSr!)super__init__rrrE read_versionr)selfrrrErargskwargs __class__s r'rKz_BaseLanceDatasink.__init__NsL $)&)))  +/.r)rcdS)NTr+rMs r'supports_distributed_writesz._BaseLanceDatasink.supports_distributed_writes`str)rct|ddddl}|jdkrB||j|j}|j|_|j|j|_dSdSdS)Nlancepylance)modulepackagerrG)r) r rUrE LanceDatasetrrversionrLr)rMrrUdss r'on_write_startz!_BaseLanceDatasink.on_write_startdsyd7I>>>> 9 ##DHd>R#SSB " D {" i ! #"r) write_resultsctddl}ddl}|s|dtdSt |dr|j}t |dkr|dtdSg}d}|D]G}|D]B\}}tj|} | | tj|}CH|sdS|j dvr|j ||} n%|j dkr|j |} |j|j| |j|jdS)Nrzwrite_results is empty. write_returnszBwrite results is empty. please check ray version or internal error>rDrHrG)rLr)warningsrUwarnDeprecationWarninghasattrr_r9pickleloadsrGrELanceOperation OverwriteAppendrYcommitrrLr) rMr]r`rUr@rbatch fragment_str schema_strr.ops r'on_write_completez$_BaseLanceDatasink.on_write_completeos    MM)"    F =/ 2 2 8)7M }   " " MMT"    F " 2 2E,1 2 2( j!< 55  ***j11 2   F 9/ / /%// BBBB Y( " "%,,Y77B !! H * 0 "     r))NrDNr!)__name__ __module__ __qualname____doc__strr r5r6r rrrKpropertyboolrSr\rr rn __classcell__rPs@r'rCrCKs(( '+9A48 // /#/56 / "$sCx.1 //////$TX ( (Xk%: (d ( ( ( (, DsCx12, , , , , , , , r)rCceZdZdZdZ ddedeejd e d d e d e d eedee ee fffd Z ede fdZdefdZdeeejdffdZxZS) LanceDatasinkafLance Ray Datasink. Write a Ray dataset to lance. If we expect to write larger-than-memory files, we can use `LanceFragmentWriter` and `LanceCommitter`. Args: uri : the base URI of the dataset. schema : pyarrow.Schema, optional. The schema of the dataset. mode : str, optional The write mode. Default is 'append'. Choices are 'append', 'create', 'overwrite'. min_rows_per_file : int, optional The minimum number of rows per file. Default is 1024 * 1024. max_rows_per_file : int, optional The maximum number of rows per file. Default is 64 * 1024 * 1024. data_storage_version: optional, str, default None The version of the data storage format to use. Newer versions are more efficient but require newer versions of lance to read. The default is "legacy" which will use the legacy v1 version. See the user guide for more details. storage_options : Dict[str, Any], optional The storage options for the writer. Default is None. LanceNrDrrrrErFmin_rows_per_filerrrctj|g|R|||d| ||_||_||_d|_dS)N)rrEr)rJrKr|rrrL) rMrrrEr|rrrrNrOrPs r'rKzLanceDatasink.__init__sw      +       "3!2$8!+/r)rc|jSr!)r|rRs r'min_rows_per_writez LanceDatasink.min_rows_per_writes %%r)c|jSr!)NAMErRs r'get_namezLanceDatasink.get_names yr)blocksrcvt||j|j|j|j|j}d|DS)N)rrrrcdg|]-\}}tj|tj|f.Sr+)rddumpsr,s r'r/z'LanceDatasink.write..sE    &\( # #V\&%9%9 :   r))rArrrrr)rMr_ctxfragments_and_schemas r'writezLanceDatasink.writesY /  H;"4!%!: 0      $8    r))NrDr{rNN)rorprqrrrrsr r5r6r intrrrKrtrrrr Tablerrvrws@r'ryrys86 D '+9A!,!1.24800 0#056 0  0  0'sm0"$sCx.10000006&C&&&X&# rx789        r)ry)rd itertoolsrtypingrrrrrr r r r pyarrowr5ray.data._internal.utilr ray.data.blockrray.data.datasource.datasinkrr0r=r1rrsrrArCryr+r)r'rs#                       111111((((((1111110//////%)-(,"*.04.:.:.: U56 7.: .: [ ! .:  .: ! .:.:#3-.:d38n-.: %/ 01.:.:.:.:bP P P P P P P P fP P P P P &P P P P P r)