
     `i                     L    d dl Z d dlmZmZ ddlmZ  G d de          ZdgZdS )    N)OptionalUnion   )PreTrainedTokenizerFastc                   h     e Zd ZdZ	 	 	 ddeeee         f         dedee         ded	e	f
 fd
Z
 xZS )ParakeetTokenizerFasta   
    Inherits all methods from [`PreTrainedTokenizerFast`]. Users should refer to this superclass for more information regarding those methods,
    except for `_decode` which is overridden to adapt it to CTC decoding:
    1. Group consecutive tokens
    2. Filter out the blank token
    FNT	token_idsskip_special_tokensclean_up_tokenization_spacesgroup_tokensreturnc                      t          |t                    r|g}|rd t          j        |          D             } fd|D             } t	                      j        d|||d|S )Nc                     g | ]
}|d          S )r    ).0token_groups     /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/parakeet/tokenization_parakeet_fast.py
<listcomp>z1ParakeetTokenizerFast._decode.<locals>.<listcomp>)   s    XXXKQXXX    c                 *    g | ]}|j         k    |S r   )pad_token_id)r   tokenselfs     r   r   z1ParakeetTokenizerFast._decode.<locals>.<listcomp>,   s&    PPPuUd>O5O5OU5O5O5Or   )r	   r
   r   r   )
isinstanceint	itertoolsgroupbysuper_decode)r   r	   r
   r   r   kwargs	__class__s   `     r   r   zParakeetTokenizerFast._decode   s     i%% 	$"I 	YXX9;LY;W;WXXXI QPPP	PPP	uww 
 3)E
 
 	
 
 	
r   )FNT)__name__
__module____qualname____doc__r   r   listboolr   strr   __classcell__)r!   s   @r   r   r      s          %*7;!
 
d3i(
 "
 '/tn	

 
 

 
 
 
 
 
 
 
 
 
r   r   )r   typingr   r   tokenization_utils_fastr   r   __all__r   r   r   <module>r-      sw         " " " " " " " " > > > > > >
 
 
 
 
3 
 
 
@ #
#r   