PiUddlZddlmZddlmZmZddlZddlZ ddl m Z ddl m Z ddlmZdd lmZerddlZddlZe Zdaeeed <Gd d eed efZdS)N)Mapping) TYPE_CHECKINGOptional)config) get_logger) map_nested)TensorFormatterDEVICE_MAPPINGceZdZdfd ZedeedffdZdZdZ dZ d efd Z d e j defd Zd e j dd fdZd e j defdZxZS) JaxFormatterNc t||ddl}ddlm}t ||r#t d|dt|dt |tr|n!t|j d|_ t| a |j ttvrtd|j dttd t|j dd t|j d|_ ||_dS) N)featurestoken_per_repo_idr)Devicez Expected z to be a `str` not z, as `jaxlib.xla_extension.Device` is not serializable neither with `pickle` nor with `dill`. Instead you can surround the device with `str()` to get its string identifier that will be internally mapped to the actual `jaxlib.xla_extension.Device`.zDevice with string identifier z) not listed among the available devices: z), so falling back to the default device: .)super__init__jaxjaxlib.xla_clientr isinstance ValueErrortypestrdevicesdevicer _map_devices_to_strlistkeysloggerwarningjnp_array_kwargs)selfrrrr#rr __class__s u/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/formatting/jax_formatter.pyrzJaxFormatter.__init__'s (>OPPP ,,,,,, ff % % ?F??tF||???  !+63 7 7RffSqAQ=R=R   !!5577N ;d>#6#6#8#899 9 9 NN444 !4!4!6!67744{s{}}Q/00444    kckmmA.//DK 0returnzjaxlib.xla_extension.Devicec<ddl}d|jDS)Nrc.i|]}t||S)r).0rs r& z4JaxFormatter._map_devices_to_str..Fs @@@F V@@@r')rr)rs r&rz JaxFormatter._map_devices_to_strBs' @@+#+--@@@@r'cddlddlm}ttr5r3t fdDr|dSS)Nrc3K|]E}t|jo+|jdjko|jdjkVFdS)rN)rArrayshapedtype)r,xcolumnrs r& z,JaxFormatter._consolidate..Mshkl 1ci((fQWq -GfAGW]^_W`WfLfr')axis)r jax.numpynumpyrrallstack)r$r4jnprs ` @r& _consolidatezJaxFormatter._consolidateHs  fd # # 1 1pv 1yyay000 r'cddl}ddlm}t|tt t dfr|St|tjtj fr8tj |j tjr| Si}t|tj tj frDtj |j tjr |jjr d|ji}n]d|ji}nSt|tj tj fr-tj |j tjr d|ji}tjr@dt,jvr2ddl}t||jjrtj|}tjr&dt,jvrddlm}t||r|Stjr*dt,jvrddlm }m!}t|||fr|StD|#a"|j$tD|j%5|j&|fii||j'cdddS#1swxYwYdS)Nrr2PIL torchvision) VideoReader torchcodec) AudioDecoder VideoDecoder)(rr7r8rrbytesrnp characterndarray issubdtyper2tolistnumberintegerrjax_enable_x64int64int32floatingfloat32 PIL_AVAILABLEsysmodules PIL.ImageImageasarrayTORCHVISION_AVAILABLEtorchvision.ior@TORCHCODEC_AVAILABLEtorchcodec.decodersrBrCr rdefault_devicerarrayr#) r$valuerr; default_dtyper>r@rBrCs r& _tensorizezJaxFormatter._tensorizeSs  ec5$t**5 6 6 "L  bj9 : : "r}U[Z\Zf?g?g "<<>> ! ebi4 5 5 3"- UWU_:`:` 3z( 5!(#) 4 !(#) 4  2:6 7 7 3BM%+WYWb>   !!5577N S t{ ; < < R R39UQQ&P&P$:O&PQQ R R R R R R R R R R R R R R R R R RsI''I+.I+cddl}tjrxdtjvrjddl}t ||jrQ| dSt|dr)t ||j s|}t |tjr2|jt$kr!fd|DSn=t |t(t*fr!fd|DS|S)Nrtorchr+ __array__c:g|]}|Sr+recursive_tensorizer, substructr$s r& z5JaxFormatter._recursive_tensorize..s()k)k)kR[$*B*B9*M*M)k)k)kr'c:g|]}|Sr+rdrfs r&rhz5JaxFormatter._recursive_tensorize..s'%g%g%gid&>&>y&I&I%g%g%gr')rrTORCH_AVAILABLErRrSrarTensorr_detachcpur8hasattrr0rbrErGr2objectr<rtuple)r$ data_structrras` r&_recursive_tensorizez!JaxFormatter._recursive_tensorizesZ   ! Og&<&< LLL+u|44 O{'9'9';';'?'?'A'A'G'G'I'I"'MNNN ; , , 2Z SY5W5W 2%//11K k2: . . i F**(()k)k)k)k_j)k)k)klll+  dE] 3 3 i$$%g%g%g%g[f%g%g%ghh h{+++r'rqc0t|j|dS)NF)map_list)r rr)r$rqs r&rez JaxFormatter.recursive_tensorizes$3[5QQQQr'pa_tablec||}|j|}||SN)numpy_arrow_extractor extract_rowpython_features_decoder decode_rowre)r$rurows r& format_rowzJaxFormatter.format_rowsK((**66x@@*55c::'',,,r' jax.Arrayc||}|j||jd}||}||}|S)Nr)rxextract_columnrz decode_column column_namesrer<)r$rur4s r& format_columnzJaxFormatter.format_columnsm++--<rs!  ######********&&&&&&''''''''''''JJJMMM !%%%%EEEEE?7K#@AEEEEEr'