
    Pig-                         d dl Z d dlmZ d dlmZ d dlZd dlZddl	m
Z
 ddlmZ erd dlZ	 d dlZdZn# e$ r d	ZdZY nw xY w G d
 deedef                   ZdS )    N)Mapping)TYPE_CHECKING   )config   )TensorFormatterTFc                        e Zd Zd fd	Zd Zd Zd ZdefdZde	j
        d	efd
Zde	j
        d	dfdZde	j
        d	efdZ xZS )TorchFormatterNc                     t                                          ||           || _        t          st	          d          d S )N)featurestoken_per_repo_idz%PyTorch is required but not available)super__init__torch_tensor_kwargs_torch_availableImportError)selfr   r   r   	__class__s       w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/formatting/torch_formatter.pyr   zTorchFormatter.__init__)   sM    (>OPPP#6  	GEFFF	G 	G    c                     t          |t                    r|s|S |d         t          t          j                  s|S t	          fd|D                       rt          j        |          S |S )z@Smarter consolidation that only stacks when safe and beneficial.r   c              3      K   | ]N}t          |t          j                  o/|j        j        k    o|j        j        k    o|j        j        k    V  Od S N)
isinstancetorchTensorshapedtypedevice).0xfirsts     r   	<genexpr>z.TorchFormatter._consolidate.<locals>.<genexpr>;   sz       
 

 	 q%,'' )5;&)5;&) EL(	
 
 
 
 
 
r   )r   listr   r   allstack)r   columnr"   s     @r   _consolidatezTorchFormatter._consolidate0   s    &$'' 	v 	M q	%.. 	M  
 
 
 

 
 
 
 
 
 	' ;v&&&r   c                    t          |t          t          t          d          f          r|S t          |t          j        t          j        f          r8t	          j        |j        t          j                  r|	                                S t          j        rdt          j        v rddl}t          ||j        j                  rt	          j        |          }|j        dk    r|ddddt          j        f         }t	          j        |dd          }|j        j        st	          j        |          }|j        j        s|                                }t3          j        |          S t          j        r&dt          j        v rddlm} t          ||          r|S t          j        r*dt          j        v rdd	lm }m!} t          |||f          r|S tE          |d
          r.t          |t2          j#                  s|$                                }t          |t          j                  rt	          j        |j        t          j%                  r| j&                                        }|'                    dt2          j(                  }|j        t          j)        t          j*        fv r|+                    t          j(                  }|t2          j(        k    r4|j        j        s|                                }t3          j        |          S |j        j        s|                                }|,                    d|           t3          j-        |fi |S |j        t          j.        k    rt	          j/        |t	          j0        t          j(                  j1        k              r|+                    t          j(                  }|t2          j(        k    r4|j        j        s|                                }t3          j        |          S |j        j        s|                                }|,                    d|           t3          j-        |fi |S |,                    d|           t3          j2        |fi |S |j        t          j(        k    rD|t2          j(        k    r4|j        j        s|                                }t3          j        |          S |j        j        s|                                }|,                    d|           t3          j-        |fi |S t	          j        |j        t          j3                  r| j&                                        }|'                    dt2          j4                  }|j        t          j4        k    rD|t2          j4        k    r4|j        j        s|                                }t3          j        |          S |j        j        s|                                }|,                    d|           t3          j-        |fi |S |j        j        s|                                }t3          j        |          S t          |t          j5                  r| j&                                        }t	          j        |j        t          j%                  r2|,                    dt2          j(                   t3          j-        |fi |S t	          j        |j        t          j3                  r2|,                    dt2          j4                   t3          j-        |fi |S t3          j-        |fi |S t          |tl          tn          f          rP	 t	          j8        |          }|j        j9        dv r| :                    |          S n# tv          tx          f$ r Y nw xY wi }	t          |tz          t|          f          r2t          |tz                    rdt2          j(        i}	ndt2          j4        i}	t3          j2        |fi i |	| j&        S )z:Zero/low-copy tensor conversion with smart dtype handling.NPILr   r   torchvision)VideoReader
torchcodec)AudioDecoderVideoDecoder	__array__r   iuf)?r   strbytestypenp	characterndarray
issubdtyper   tolistr   PIL_AVAILABLEsysmodules	PIL.ImageImageasarrayndimnewaxismoveaxisflagsc_contiguousascontiguousarray	writeablecopyr   
from_numpyTORCHVISION_AVAILABLEtorchvision.ior-   TORCHCODEC_AVAILABLEtorchcodec.decodersr/   r0   hasattrr   r1   integerr   getint64uint16uint32astype
setdefault	as_tensoruint64r%   iinfomaxtensorfloatingfloat32numberr$   tuplearraykind
_tensorize
ValueError	TypeErrorintfloat)
r   valuer*   arrr-   r/   r0   kwargstarget_dtypedefault_dtypes
             r   ra   zTorchFormatter._tensorizeF   s>    ec5$t**566 	L eblBJ788 	"R]5;XZXd=e=e 	"<<>>!  	-ES[$8$8%11 -j''8q==aaaBJ./Ck#r1--y- 4.s33Cy* %((**C',,, ' 	MS[,H,H222222%-- & 	<3;+F+FFFFFFFFF%,!=>>  5+&& 	&z%/N/N 	&OO%%E eRZ(( _	}U["*55 F/16688%zz'5;?? ;29bi"888!LL22E#u{22${4 1$)JJLLE$/666${4 1$)JJLLE))'<@@@$u?????[BI--verx'9'9'==>> = %RX 6 6'5;66#(;#8 5(-

#(#3E#:#::#(;#8 5(-

"--g|DDD#(?5#C#CF#C#CC ))'<@@@$|E<<V<<< {bh..<5;3N3N${4 1$)JJLLE$/666  %{4 1$)JJLLE))'<@@@$u????? u{BK88 /16688%zz'5=AA;"*,,1N1N ;0 - %

 +E222 !;0 - %

%%g|<<< ?5;;F;;; {, )!JJLLE'... ry)) 	-2244F}U["*55 8!!'5;777u77777u{BK88 8!!'5=999u77777u77777 e}-- 	huoo9>U**??3/// +	*    ec5\** 	9%%% 9!(%+ 6!(%- 8|ESS%R%R9Q%RSSSs   6^	 	^^c                 .    t          |d          r.t          |t          j                  s|                                }t          |t
          j                  r4|j        t          k    r# fd|D             } 	                    |          S ntt          |t          t          f          r# fd|D             } 	                    |          S t          |t                    r  fd|                                D             S                      |          S )z8Optimized recursive walker with reduced Python overhead.r1   c                 :    g | ]}                     |          S  _recursive_tensorizer    itemr   s     r   
<listcomp>z7TorchFormatter._recursive_tensorize.<locals>.<listcomp>   s'    RRRd$33D99RRRr   c                 :    g | ]}                     |          S rm   rn   rp   s     r   rr   z7TorchFormatter._recursive_tensorize.<locals>.<listcomp>   s'    NNN$d//55NNNr   c                 B    i | ]\  }}|                     |          S rm   rn   )r    keyrf   r   s      r   
<dictcomp>z7TorchFormatter._recursive_tensorize.<locals>.<dictcomp>   s-    ```jc5C22599```r   )rN   r   r   r   r1   r6   r8   r   objectr(   r$   r^   dictitemsra   )r   data_structresults   `  r   ro   z#TorchFormatter._recursive_tensorize   s%    ;,, 	2ZU\5Z5Z 	2%//11K k2:.. 	a F**RRRRkRRR((000 +
 dE]33 	aNNNN+NNNF$$V,,,T** 	a````KL]L]L_L_```` {+++r   rz   c                 ,    |                      |          S )z+Public interface maintaining compatibility.rn   )r   rz   s     r   recursive_tensorizez"TorchFormatter.recursive_tensorize   s    ((555r   pa_tablereturnc                     |                                                      |          }| j                            |          }|                     |          S r   )numpy_arrow_extractorextract_rowpython_features_decoder
decode_rowr}   )r   r~   rows      r   
format_rowzTorchFormatter.format_row   sK    ((**66x@@*55c::'',,,r   torch.Tensorc                     |                                                      |          }| j                            ||j        d                   }|                     |          }|                     |          }|S )Nr   )r   extract_columnr   decode_columncolumn_namesr}   r(   )r   r~   r'   s      r   format_columnzTorchFormatter.format_column   sm    ++--<<XFF-;;FHDYZ[D\]]))&11""6**r   c                     |                                                      |          }| j                            |          }|                     |          }|D ] }|                     ||                   ||<   !|S r   )r   extract_batchr   decode_batchr}   r(   )r   r~   batchcolumn_names       r   format_batchzTorchFormatter.format_batch  s    **,,::8DD,99%@@((//  	G 	GK!%!2!253E!F!FE+r   )NN)__name__
__module____qualname__r   r(   ra   ro   rx   r}   paTabler   r   r   r   __classcell__)r   s   @r   r
   r
   (   s        G G G G G G  ,XT XT XTt, , ,.6t 6 6 6 6-28 - - - - -
bh >    RX '        r   r
   r   )r<   collections.abcr   typingr   numpyr6   pyarrowr    r   
formattingr   r   r   r   r
   rm   r   r   <module>r      s    


 # # # # # #                           ' ' ' ' ' '  LLLLLL   EEE
e e e e e_Wng%EF e e e e es   3 	??