
    Pit                         d dl Z d dlmZ d dlmZ d dlZd dlZddl	m
Z
 ddlmZ ddlmZ erd dlZ G d	 d
eedef                   ZdS )    N)Mapping)TYPE_CHECKING   )config)
map_nested   )TensorFormatterc                        e Zd Zd fd	Zd Zd Zd ZdefdZde	j
        d	efd
Zde	j
        d	dfdZde	j
        d	efdZ xZS )TFFormatterNc                 d    t                                          ||           || _        dd l}d S )N)featurestoken_per_repo_idr   )super__init__tf_tensor_kwargs
tensorflow)selfr   r   r   tf	__class__s        t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/formatting/tf_formatter.pyr   zTFFormatter.__init__!   s;    (>OPPP 0    c                    dd l t          t                    rdrbt          fdD                       r j                  S t          fdD                       rj                                      S S )Nr   c              3      K   | ]E}t          |j                  o+|j        d          j        k    o|j        d          j        k    V  FdS )r   N)
isinstanceTensorshapedtype.0xcolumnr   s     r   	<genexpr>z+TFFormatter._consolidate.<locals>.<genexpr>*   sh        kl
1bi((fQWq	-GfAGW]^_W`WfLf     r   c              3      K   | ]A}t          |j        j        f          o |j        d k    o|j        d         j        k    V  BdS )r   r   N)r   r   RaggedTensorndimr   r   s     r   r"   z+TFFormatter._consolidate.<locals>.<genexpr>.   sm         1ry"/:;;j!jPQPW[abc[d[jPj     r   )r   r   listallstackragged)r   r!   r   s    `@r   _consolidatezTFFormatter._consolidate&   s    fd## 
	/ 
	/     pv     	/  rx'''          /
 yv...r   c                    dd l }||S i }t          |t          j        t          j        f          r.t          j        |j        t          j                  r
d|j        i}nSt          |t          j        t          j        f          r-t          j        |j        t          j	                  r	d|j
        i}t          j        r@dt          j        v r2dd l}t          ||j        j                  rt          j        |          }t          j        r&dt          j        v rddlm} t          ||          r|S t          j        r*dt          j        v rddlm}m} t          |||f          r|S  |j        |fi i || j        S )Nr   r   PILtorchvision)VideoReader
torchcodec)AudioDecoderVideoDecoder)r   r   npnumberndarray
issubdtyper   integerint64floatingfloat32r   PIL_AVAILABLEsysmodules	PIL.ImageImageasarrayTORCHVISION_AVAILABLEtorchvision.ior.   TORCHCODEC_AVAILABLEtorchcodec.decodersr0   r1   convert_to_tensorr   )r   valuer   default_dtyper,   r.   r0   r1   s           r   
_tensorizezTFFormatter._tensorize7   s   =Lebi455 	2"-UWU_:`:` 	2$bh/MM	2:677 	2BM%+WYWb<c<c 	2$bj1M 	*ES[$8$8%11 *
5))' 	MS[,H,H222222%-- & 	<3;+F+FFFFFFFFF%,!=>> #r#EXX-W-WAV-WXXXr   c                     dd l }t          j        rxdt          j        v rjdd l}t          ||j                  rQ                     |	                                
                                                                d                   S t          |d          r)t          ||j                  s|                                }t          |t          j                  r2|j        t"          k    r!                      fd|D                       S n=t          |t&          t(          f          r!                      fd|D                       S                      |          S )Nr   torch 	__array__c                 :    g | ]}                     |          S rJ   recursive_tensorizer   	substructr   s     r   
<listcomp>z4TFFormatter._recursive_tensorize.<locals>.<listcomp>d   s(    )k)k)kR[$*B*B9*M*M)k)k)kr   c                 :    g | ]}                     |          S rJ   rM   rO   s     r   rQ   z4TFFormatter._recursive_tensorize.<locals>.<listcomp>f   s'    %g%g%gid&>&>y&I&I%g%g%gr   )r   r   TORCH_AVAILABLEr;   r<   rI   r   r   rG   detachcpunumpyhasattrrK   r2   r4   r   objectr*   r&   tuple)r   data_structr   rI   s   `   r   _recursive_tensorizez TFFormatter._recursive_tensorizeV   s]    ! 	Og&<&<LLL+u|44 O{'9'9';';'?'?'A'A'G'G'I'I"'MNNN;,, 	2ZRY5W5W 	2%//11Kk2:.. 	i F**(()k)k)k)k_j)k)k)klll +dE]33 	i$$%g%g%g%g[f%g%g%ghhh{+++r   rZ   c                 0    t          | j        |d          S )NF)map_list)r   r[   )r   rZ   s     r   rN   zTFFormatter.recursive_tensorizei   s    $3[5QQQQr   pa_tablereturnc                     |                                                      |          }| j                            |          }|                     |          S N)numpy_arrow_extractorextract_rowpython_features_decoder
decode_rowrN   )r   r^   rows      r   
format_rowzTFFormatter.format_rowl   sK    ((**66x@@*55c::'',,,r   	tf.Tensorc                     |                                                      |          }| j                            ||j        d                   }|                     |          }|                     |          }|S )Nr   )rb   extract_columnrd   decode_columncolumn_namesrN   r*   )r   r^   r!   s      r   format_columnzTFFormatter.format_columnq   sm    ++--<<XFF-;;FHDYZ[D\]]))&11""6**r   c                     |                                                      |          }| j                            |          }|                     |          }|D ] }|                     ||                   ||<   !|S ra   )rb   extract_batchrd   decode_batchrN   r*   )r   r^   batchcolumn_names       r   format_batchzTFFormatter.format_batchx   s    **,,::8DD,99%@@((//  	G 	GK!%!2!253E!F!FE+r   )NN)__name__
__module____qualname__r   r*   rG   r[   dictrN   paTabler   rg   rm   rs   __classcell__)r   s   @r   r   r       s                   
  "Y Y Y>, , ,&Rt R R R R-28 - - - - -
bh ;    RX '        r   r   rh   )r;   collections.abcr   typingr   rV   r2   pyarrowrx    r   utils.py_utilsr   
formattingr	   r   r   r   rJ   r   r   <module>r      s     


 # # # # # #                           ' ' ' ' ' ' ' ' ' ' ' '  ^ ^ ^ ^ ^/';"?@ ^ ^ ^ ^ ^r   