
    Pi                         d dl mZmZ d dlmZmZmZmZmZ d dl	Z
erddlmZ e G d d                      Ze G d d	                      ZdS )
    )	dataclassfield)TYPE_CHECKINGAnyClassVarOptionalUnionN   )FeatureTypec                       e Zd ZU dZee         ed<    edd          Ze	e         ed<   dZ
ee         ed<   dZee         ed	<    ed dd
          Zeed<   d Zdedeedf         f         fdZdS )Translationa  `Feature` for translations with fixed languages per example.
    Here for compatibility with tfds.

    Args:
        languages (`dict`):
            A dictionary for each example mapping string language codes to string translations.

    Example:

    ```python
    >>> # At construction time:
    >>> datasets.features.Translation(languages=['en', 'fr', 'de'])
    >>> # During data generation:
    >>> yield {
    ...         'en': 'the cat',
    ...         'fr': 'le chat',
    ...         'de': 'die katze'
    ... }
    ```
    	languagesNFdefaultrepriddictdtypepa_typer   initr   _typec                 b    t          j        d t          | j                  D                       S )Nc                 6    i | ]}|t          j                    S  )pastring).0langs     q/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/features/translation.py
<dictcomp>z(Translation.__call__.<locals>.<dictcomp>*   s     OOO$	OOO    )r   structsortedr   selfs    r    __call__zTranslation.__call__)   s+    yOOt~8N8NOOOPPPr"   returnr   c                 P    ddl m fdt          | j                  D             S )z2Flatten the Translation feature into a dictionary.r
   )Valuec                 (    i | ]}| d           S )r   r   )r   kr*   s     r    r!   z'Translation.flatten.<locals>.<dictcomp>0   s#    CCCq55??CCCr"   )featuresr*   r$   r   )r&   r*   s    @r    flattenzTranslation.flatten,   s9    ######CCCCF4>,B,BCCCCr"   )__name__
__module____qualname____doc__liststr__annotations__r   r   r   r   r   r   r   r   r'   r	   r   r.   r   r"   r    r   r      s          * Cyd777B777!E8C=!!!!GXc]!!!}5uEEEE3EEEQ Q QD}d33E.FFG D D D D D Dr"   r   c                      e Zd ZU dZdZee         ed<   dZee	         ed<    e
dd          Zee         ed<   dZee         ed	<   dZee         ed
<    e
d dd          Zeed<   d Zd Zd Zdedeedf         f         fdZdS )TranslationVariableLanguagesa  `Feature` for translations with variable languages per example.
    Here for compatibility with tfds.

    Args:
        languages (`dict`):
            A dictionary for each example mapping string language codes to one or more string translations.
            The languages present may vary from example to example.

    Returns:
        - `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`):
            Language codes sorted in ascending order or plain text translations, sorted to align with language codes.

    Example:

    ```python
    >>> # At construction time:
    >>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de'])
    >>> # During data generation:
    >>> yield {
    ...         'en': 'the cat',
    ...         'fr': ['le chat', 'la chatte,']
    ...         'de': 'die katze'
    ... }
    >>> # Tensor returned :
    >>> {
    ...         'language': ['en', 'de', 'fr', 'fr'],
    ...         'translation': ['the cat', 'die katze', 'la chatte', 'le chat'],
    ... }
    ```
    Nr   num_languagesFr   r   r   r   r   r   r   c                     | j         r!t          t          | j                             nd | _         | j         rt          | j                   nd | _        d S )N)r   r$   setlenr8   r%   s    r    __post_init__z*TranslationVariableLanguages.__post_init__\   sK    8<PDN 3 3444D48NLS000r"   c                     t          j        t          j        t          j                              t          j        t          j                              d          S )Nlanguagetranslation)r   r#   list_r   r%   s    r    r'   z%TranslationVariableLanguages.__call__`   s>    ybhry{{&;&;BHUWU^U`U`LaLabbcccr"   c           
      N   t          | j                  }t          |          ddhk    r|S | j        rkt          |          |z
  rYt          dd                    t	          t          |          |z
                       dd                    |           d          g }|                                D ]S\  }t          |t                    r|                    |f           2|	                    fd|D                        Tt          t	          |           \  }}||dS )	Nr?   r@   zSome languages in example (z, z) are not in valid set (z).c                     g | ]}|fS r   r   )r   elr   s     r    
<listcomp>z?TranslationVariableLanguages.encode_example.<locals>.<listcomp>s   s    *E*E*E"D":*E*E*Er"   r>   )r:   r   
ValueErrorjoinr$   items
isinstancer4   appendextendzip)r&   translation_dictlang_settranslation_tuplestextr   translationsr   s          @r    encode_examplez+TranslationVariableLanguages.encode_examplec   s|   t~&&  Z$???##^ 	$4 5 5 @ 	 SdiisCS?T?TW_?_8`8`.a.a  S  S{  |E  |E  FN  |O  |O  S  S  S    *0022 	G 	GJD$$$$ G"))4,7777"))*E*E*E*E*E*E*EFFFF #&v.@'A'A"B	<%lCCCr"   r(   r   c                 d    ddl m}m}  | |d                     | |d                    dS )zCFlatten the TranslationVariableLanguages feature into a dictionary.r
   )Listr*   r   r>   )r-   rT   r*   )r&   rT   r*   s      r    r.   z$TranslationVariableLanguages.flattenz   sU    )))))))) UU8__--4h00
 
 	
r"   )r/   r0   r1   r2   r   r   r3   r5   r8   intr   r   r4   r   r   r   r   r   r<   r'   rR   r	   r   r.   r   r"   r    r7   r7   3   s         > !%Ix~$$$#'M8C='''d777B777!E8C=!!!!GXc]!!!=EPUVVVE3VVVM M Md d dD D D.
}d33E.FFG 
 
 
 
 
 
r"   r7   )dataclassesr   r   typingr   r   r   r   r	   pyarrowr   r-   r   r   r7   r   r"   r    <module>rY      s    ( ( ( ( ( ( ( ( @ @ @ @ @ @ @ @ @ @ @ @ @ @      &%%%%%% $D $D $D $D $D $D $D $DN M
 M
 M
 M
 M
 M
 M
 M
 M
 M
r"   