
    h6s                         d dl Z d dlZd dlmZmZmZmZmZmZ d dl	Z	d dl
Z
d dlZd dlZd dlmZmZmZmZmZ d dlmZmZ ddlmZ  ej        e          Z G d d          ZdgZdS )	    N)ListOptionalUnionDictAnyTuple)BatchEncodingPaddingStrategyPreTokenizedInput	TextInputTruncationStrategy)
TensorTypelogging   )AudioNormalizerc                      e Zd ZdZd0dZed             Zdeee	j
        f         fdZ	 	 	 	 	 	 	 d1d
eeeee         eeee         ee         f                  deeeeeej        f                  eeeeej        f                           f                  deeeef         deeeef         dee         deeeef                  dedefdZ	 d2d
eeef         deeeeej        f                           deeef         fdZ	 	 	 	 	 d3deeeef                  deeeef         deeeef         dee         deeeef                  dedefdZdeeeej        f                  deee         eej                 ee         f         fdZ	 	 	 d4deej                 deeeef                  deeeej         f                  deej!                 deeef         f
dZ"dedefdZ#dedefdZ$d edeeeef                  fd!Z%d"ed#edefd$Z&d% Z'd& Z(e)d'             Z*	 	 	 	 d5d*eej+        ej        eeej+        ej        f                  f         d+ed,ee         d-ed.edefd/Z,dS )6VibeVoiceProcessoraW  
    Constructs a VibeVoice processor which wraps a VibeVoice tokenizer and audio processor into a single processor.

    [`VibeVoiceProcessor`] offers all the functionalities of [`VibeVoiceTokenizer`] and [`VibeVoiceTokenizerProcessor`]. 
    See the [`~VibeVoiceProcessor.__call__`] and [`~VibeVoiceProcessor.decode`] for more information.

    Args:
        tokenizer (`VibeVoiceTextTokenizer` or `VibeVoiceTextTokenizerFast`):
            The tokenizer for text processing.
        audio_processor (`VibeVoiceTokenizerProcessor`):
            The audio processor for speech processing.
        speech_tok_compress_ratio (`int`, *optional*, defaults to 3200):
            The compression ratio for speech tokenization.
        db_normalize (`bool`, *optional*, defaults to True):
            Whether to apply decibel normalization to audio inputs.
    N  Tc                 z    || _         || _        || _        || _        |rt	                      nd | _        d| _        d S )Nz~ Transform the text provided by various speakers into speech output, utilizing the distinct voice of each respective speaker.
)	tokenizeraudio_processorspeech_tok_compress_ratiodb_normalizer   audio_normalizersystem_prompt)selfr   r   r   r   kwargss         O/workspace/chatterbox-finetuning/src/vibevoice/processor/vibevoice_processor.py__init__zVibeVoiceProcessor.__init__#   sJ    ".)B&(5A K 1 1 1t _    c           	         ddl }ddl}ddlm} ddlm}m} |j                            |d          }|j        	                    |          r>t          |d          5 }	|                    |	          }
ddd           n# 1 swxY w Y   n#t                              d| d	           d
dd}
|
                    dd
          }|
                    dd          }|
                    dd          p|                    dd          }t                              d|            d|                                v r |j        |fi |}nt'          d| d          d|
v rh|
d         } ||                    dd          |                    dd          |                    dd          |                    dd                    }n
 |            } | ||||          S )a  
        Instantiate a VibeVoiceProcessor from a pretrained VibeVoice processor.

        Args:
            pretrained_model_name_or_path (`str` or `os.PathLike`):
                This can be either:
                - a string, the *model id* of a pretrained model
                - a path to a *directory* containing processor config

        Returns:
            [`VibeVoiceProcessor`]: The processor object instantiated from pretrained model.
        r   Nr   )VibeVoiceTokenizerProcessor)VibeVoiceTextTokenizerVibeVoiceTextTokenizerFastpreprocessor_config.jsonrz%No preprocessor_config.json found at z, using defaultsr   T)r   r   r   r   language_model_pretrained_namezQwen/Qwen2.5-1.5BzLoading tokenizer from qwenzUnsupported tokenizer type for z&. Supported types: Qwen, Llama, Gemma.r   sampling_rate]  normalize_audiotarget_dB_FSepsư>)r)   r+   r,   r.   )r   r   r   r   )osjsonvibevoice_tokenizer_processorr"   2vibevoice.modular.modular_vibevoice_text_tokenizerr#   r$   pathjoinexistsopenloadloggerwarninggetpopinfolowerfrom_pretrained
ValueError)clspretrained_model_name_or_pathr   r0   r1   r"   r#   r$   config_pathfconfigr   r   r'   r   audio_configr   s                    r   r?   z"VibeVoiceProcessor.from_pretrained+   s    				NNNNNN	
 	
 	
 	
 	
 	
 	
 	
 gll#@B\]]7>>+&& 	k3'' &11& & & & & & & & & & & & & & & NNrC`rrrsss-1 $ F %+JJ/JD$Q$Q!zz.$77 *04TVZ)[)[  *a_e_i_i  kK  M`  `a  `a&N.LNNOOO399;;;;B2B.  II
   F?]  F  F  F  G  G  G &&!"34L99*..FF , 0 01BD I I)--ncBB $$UD11	  OO :9;;O s+&?%	
 
 
 	
s   A>>BBsave_directoryc                    ddl }ddl} |j        |d           d| j        | j        dt          | j        dd          t          | j        d	d          t          | j        d
d          t          | j        dd          dd}|j                            |d          }t          |d          5 }|
                    ||d           ddd           n# 1 swxY w Y   t                              d|            dS )a  
        Save a processor to a directory, so that it can be re-loaded using the
        [`~VibeVoiceProcessor.from_pretrained`] class method.

        Args:
            save_directory (`str` or `os.PathLike`):
                Directory where the processor will be saved.
        r   NT)exist_okr   r"   r)   r*   r+   r,   r-   r.   r/   )feature_extractor_typer)   r+   r,   r.   )processor_classr   r   r   r%   w   )indentz!Processor configuration saved in )r0   r1   makedirsr   r   getattrr   r4   r5   r7   dumpr9   r=   )r   rG   r   r0   r1   processor_configrC   rD   s           r   save_pretrainedz"VibeVoiceProcessor.save_pretrainedr   sY    				NT2222  4)-)G -*G!()=PU!V!V#*4+?ARTX#Y#Y '(<nc R Rt3UDAA   	
 
 gll>3MNN+s## 	5qII&!I444	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	EEEFFFFFs   -CCCFtextvoice_samplespadding
truncation
max_lengthreturn_tensorsreturn_attention_maskreturnc                    t          |t                    sCt          |t                    r4t          |          dk    r!t          |d         t                    s|g}	d}
n|}	d}
|0|
r't          |d         t          t          j        f          r|g}n|}ndgt          |	          z  }g }t          |	|          D ]0\  }}|                     ||          }|                    |           1| 	                    ||||||          }|S )a  
        Main method to process one or more podcast scripts with optional voice samples.

        Args:
            text (`str`, `List[str]`):
                The input text(s) to process. Can be:
                - A single script string
                - A list of script strings for batch processing
                - A path to a .json or .txt file
                - A list of paths
            voice_samples (`List[Union[str, np.ndarray]]`, `List[List[Union[str, np.ndarray]]]`, *optional*):
                Voice samples for each script. Can be:
                - A list of samples for a single script
                - A list of lists for batch processing
            padding (`bool`, `str` or `PaddingStrategy`, defaults to `True`):
                Whether to pad sequences to the same length
            truncation (`bool`, `str` or `TruncationStrategy`, defaults to `False`):
                Whether to truncate sequences
            max_length (`int`, *optional*):
                Maximum length of the returned sequences
            return_tensors (`str` or `TensorType`, *optional*):
                If set, will return tensors of a particular framework
            return_attention_mask (`bool`, defaults to `True`):
                Whether to return the attention mask

        Returns:
            `BatchEncoding`: A BatchEncoding with the following fields:
                - **input_ids** -- List of token id sequences or tensor
                - **attention_mask** -- List of attention masks or tensor
                - **speech_tensors** -- Padded speech inputs (if voice_samples provided)
                - **speech_masks** -- Speech masks (if voice_samples provided)
                - **speech_input_mask** -- Boolean masks indicating speech token positions
        r   FTN)rV   rW   rX   rY   rZ   )

isinstancestrlistlennpndarrayzip_process_singleappend_batch_encode)r   rT   rU   rV   rW   rX   rY   rZ   r   texts
is_batchedvoice_samples_listall_encodings
text_inputvoice_inputencodingbatch_encodings                    r   __call__zVibeVoiceProcessor.__call__   sE   Z dC   	Zd%;%; 	D		AV`aefgahjmVnVnFEJJ EJ $ 3*]1-=RZ?P"Q"Q 3&3_"" &3"""&#e**!4 '*52D'E'E 	+ 	+#J++JDDH  **** ++!!)"7 , 
 
 r    c                    d}t          |t                    r|                    d          r5t          j                            |          r|                     |          }nL|                    d          r5t          j                            |          r|                     |          }n|}|t          d|           | 	                    |          }t          t          d |D                                 }| j                            | j                  }|r/|                     |dt!          |                             \  }}}	ng g g }	}}||z   }
dgt!          |          z  |	z   }|
| j                            dd          z  }
|dgt!          | j                            dd                    z  z  }|D ]C\  }}| j                            d	| d
| dd          }|
|z  }
|dgt!          |          z  z  }D|
| j                            dd          | j        j        gz   z  }
|dgt!          | j                            dd                    dz   z  z  }|
|r|nd|||dS )z Process a single podcast script.Nz.jsonz.txtzCould not process input text: c              3       K   | ]	\  }}|V  
d S N ).0
speaker_id_s      r   	<genexpr>z5VibeVoiceProcessor._process_single.<locals>.<genexpr>   s&      MM}z1
MMMMMMr    Fz Text input:
add_special_tokens	 Speaker :
z Speech output:
r   )	input_idsspeech_inputsspeech_input_maskparsed_scriptall_speakers)r]   r^   endswithr0   r4   r6   _convert_json_to_script_convert_text_to_scriptr@   _parse_scriptr_   setr   encoder   _create_voice_promptr`   speech_start_id)r   rT   rU   scriptparsed_linesr   system_tokensvoice_tokensvoice_speech_inputsvoice_speech_masksfull_tokensr   ru   speaker_textspeaker_text_tokenss                  r   rd   z"VibeVoiceProcessor._process_single   s    dC   	}}W%% "'..*>*> 55d;;v&& 27>>$+?+? 55d;; >DdDDEEE ))&11CMMMMMMMNN --d.@AA  	ODHD]D]^kl~mpq}m~m~l~^  EA  EAAL-/A/ADFB/A-L $l2"Gc-&8&88;MM 	t~,,-=RW,XXXeWs4>+@+@AQfk+@+l+l'm'mmm(4 	D 	D$J"&."7"78aJ8a8aQ]8a8a8av{"7"|"|..K%3/B+C+C!CC 	t~,,-@UZ,[[_c_m_}^~~~eWDN,A,ABUjo,A,p,p(q(qtu(uvv %4GQ00T!2)(
 
 	
r    	encodingsc                    d |D             }d |D             }t          |t                    r|rt          j        nt          j        }	n't          |t
                    rt          |          }	n|}	|	t          j        k    r)|	t          j        k    rt          d |D                       }
n.|	t          j        k    r||}
nt          d |D                       }
g }g }g }t          ||          D ]\  }}|r't          |          |
k    r|d|
         }|d|
         }|
t          |          z
  }| j
        j        g|z  |z   }dg|z  dgt          |          z  z   }dg|z  |z   }|                    |           |                    |           |                    |           |}|}n|rd	 |D             nd}g }d}|D ]'}|d
         |                    |d
                    d}(t                      }|nt          j        |t          j                  |d<   |r%|#t          j        |t          j                  |d<   t          j        |t          j                  |d<   n||d<   |r|||d<   ||d<   |r.|                     ||          }|d         |d<   |d         |d<   n
d|d<   d|d<   d |D             |d<   d |D             |d<   |S )z5Combine multiple encodings into a batch with padding.c                     g | ]
}|d          S )r}   rs   rt   encs     r   
<listcomp>z4VibeVoiceProcessor._batch_encode.<locals>.<listcomp>.  s    @@@s#k*@@@r    c                     g | ]
}|d          S )r   rs   r   s     r   r   z4VibeVoiceProcessor._batch_encode.<locals>.<listcomp>/  s    "Q"Q"Q3':#;"Q"Q"Qr    c              3   4   K   | ]}t          |          V  d S rr   r`   rt   idss     r   rw   z3VibeVoiceProcessor._batch_encode.<locals>.<genexpr><  (      AA3c#hhAAAAAAr    Nc              3   4   K   | ]}t          |          V  d S rr   r   r   s     r   rw   z3VibeVoiceProcessor._batch_encode.<locals>.<genexpr>@  r   r    r   r   Fc                 4    g | ]}d gt          |          z  S )r   r   r   s     r   r   z4VibeVoiceProcessor._batch_encode.<locals>.<listcomp>\  s$    HHH#sSXX~HHHr    r~   Tdtyper}   attention_maskr   )rY   padded_speechesspeech_tensorsspeech_masksc                     g | ]
}|d          S )r   rs   r   s     r   r   z4VibeVoiceProcessor._batch_encode.<locals>.<listcomp>  s    +V+V+VSC,@+V+V+Vr    parsed_scriptsc                     g | ]
}|d          S )r   rs   r   s     r   r   z4VibeVoiceProcessor._batch_encode.<locals>.<listcomp>  s    .X.X.Xss>/B.X.X.Xr    all_speakers_list)r]   boolr
   LONGEST
DO_NOT_PADr^   max
MAX_LENGTHrc   r`   r   pad_idre   extendr	   torchtensorlongprepare_speech_inputs)r   r   rV   rW   rX   rY   rZ   input_ids_listspeech_input_masks_listpadding_strategymax_lenpadded_input_idsattention_maskspadded_speech_input_masksr}   speech_maskpadding_length
padded_idsr   padded_speech_maskall_speech_inputs
has_speechr   rn   speech_dicts                            r   rf   z VibeVoiceProcessor._batch_encode#  s    A@i@@@"Q"Qy"Q"Q"Q gt$$ 	':Aa66Ga%% 	'.w77& 999?#:::AA.AAAAA!_%???JDZ$AA.AAAAA  " O(*%*-n>U*V*V E E&	; 8#i..7":": )(7( 3I"-hwh"7K ")3y>>!9"n34~E	Q
"#~!5c)nn8L!L&+W~%=%K" ''
333&&~666)001CDDDD-N&?## MbkHHHHHHgkO 
 	" 	"C?#/!((_)=>>>!
 ' %*/,~UZ*X*X*XN;'$ c)D38<W\Wa3b3b3b/027,?V^c^h2i2i2iN.//*8N;'$ C)D3B/02IN./  		244!- 5  K 0;;L/MN+,-8-HN>**/3N+,-1N>* ,W+VI+V+V+V'(.X.Xi.X.X.X*+r    speaker_samplesc                 v   | j         j        }| j                             dd          }g }dgt          |          z  }t	          |          D ]g\  }}| j                             d| dd          }t          |t                    r| j                            |          }	n t          j
        |t          j                  }	| j        r| j        r|                     |	          }	t          j        |	j        d         | j        z            }
|| j         j        gz   |g|
z  z   | j         j        gz   | j                             dd          z   }dgt          |          z  dgz   d	g|
z  z   dgz   dgz   }|                    |           |                    |           |                    |	           i|||fS )
z
        Create voice prompt tokens and process audio samples.
        
        Returns:
            tuple: (voice_tokens, voice_speech_inputs, voice_speech_masks)
        z Voice input:
Frx   rz   r{   r   r   r|   T)r   speech_diffusion_idr   r`   	enumerater]   r^   r   _load_audio_from_pathra   arrayfloat32r   r   mathceilshaper   r   speech_end_idr   re   )r   r   vae_token_idvoice_full_tokensr   r   ru   speaker_audioprefix_tokenswavvae_tok_lenspeaker_tokensvae_input_masks                r   r   z'VibeVoiceProcessor._create_voice_prompt  s    ~9 N112CX]1^^ #Ws+<'='==)2?)C)C #	, #	,%J N112Kj2K2K2K`e1ffM --- @*@@OOh}BJ???   1T%: 1++C00 )CIaL43Q$QRRK ,!^;<=)N[89 "^9:; !N11$51QQ	RN  %gM(:(::"G$!F[01 #G$ #G	$N $$^444%%n555&&s++++ "57IIIr    r~   devicer   c                     |sdddS  fd|D             }t          d |D                       }|d         j        dk    r1t          j        t	          |          |fdt          j                  }nBt          j        t	          |          ||d         j        d         fdt          j                  }t          j        t	          |          t          |          ft          j        	          }t          t          ||                    D ]'\  }	\  }
}|
||	dt	          |
          f<   d
||	d|f<   (||d}|dk    rJt          j        |||pt          j                  |d<   t          j        ||t          j                  |d<   |S )ah  
        Prepare speech inputs for model consumption.
        
        Args:
            speech_inputs: List of speech arrays
            return_tensors: Output tensor type
            device: Device to place tensors on
            dtype: Data type for tensors
            
        Returns:
            Dictionary with padded_speeches and speech_masks
        N)r   r   c                 ^    g | ])}t          j        |j        d          j        z            *S )r   )r   r   r   r   )rt   sr   s     r   r   z<VibeVoiceProcessor.prepare_speech_inputs.<locals>.<listcomp>  s2    iiiVW49QWQZ$2P%PQQiiir    c              3   0   K   | ]}|j         d          V  dS )r   N)r   )rt   r   s     r   rw   z;VibeVoiceProcessor.prepare_speech_inputs.<locals>.<genexpr>  s(      BBq
BBBBBBr    r   r   )
fill_valuer   r   Tpt)r   r   r   r   )r   ndimra   fullr`   r   r   zerosbool_r   rc   r   r   r   )r   r~   rY   r   r   vae_tok_seqlensmax_speech_lengthr   r   ispeechvae_tok_lengthresults   `            r   r   z(VibeVoiceProcessor.prepare_speech_inputs  s   &  	C'+TBBB jiii[hiiiBBMBBBBB  A%% gs='9'9;L&MZ[cecmnnnOO gs='9'9;Lm\]N^NdegNh&ivw  @B  @J  K  K  KOx]!3!3S5I5I JRTRZ[[[+4S5X5X+Y+Y 	4 	4'A'/5OA|F|O,/3LO^O+,,  /(
 
 T!!(-_V[`[qdidq(r(r(rF$%%*\,vUZU_%`%`%`F>"r    	json_filec                    ddl }t          |dd          5 }|                    |          }ddd           n# 1 swxY w Y   t          |t                    st          d          g }|D ]}t          |t                    st                              d|            5|	                    d          }|	                    d	          }||t                              d
|            	 t          |          }	n5# t
          t          f$ r! t                              d| d           Y w xY w|                                }|r|                    d|	 d|            |st          d          d                    |          S )z
        Convert JSON format to script format.
        Expected JSON format:
        [
            {"speaker": "1", "text": "Hello everyone..."},
            {"speaker": "2", "text": "Great to be here..."}
        ]
        r   Nr&   utf-8rm   z0JSON file must contain a list of speaker entrieszSkipping non-dict entry: speakerrT   z(Skipping entry missing speaker or text: zInvalid speaker ID: z, skipping entrySpeaker : z#No valid entries found in JSON filer|   )r1   r7   r8   r]   r_   r@   dictr9   r:   r;   int	TypeErrorstripre   r5   )
r   r   r1   rD   datascript_linesitemr   rT   ru   s
             r   r   z*VibeVoiceProcessor._convert_json_to_script  s    	)S7333 	 q99Q<<D	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  $%% 	QOPPP 	E 	EDdD)) A4AABBBhhy))G88F##D$,P$PPQQQ \\

	*   OgOOOPPP
 ::<<D E##$Cz$C$CT$C$CDDD 	DBCCCyy&&&s   9= =.C>>/D0/D0	text_filec                 R   t          |dd          5 }|                                }ddd           n# 1 swxY w Y   g }d}|D ]}|                                }|st          j        d|t          j                  }|rgt          |                    d                    }|                    d                                          }	|	r|                    d| d	|	            |                    d| d	|            |st          d
          d
                    |          S )z
        Convert text file to script format.
        Handles multiple formats:
        1. Already formatted as "Speaker X: text"
        2. Plain text (assigns to Speaker 1)
        
        Handles edge cases like multiple colons in a line.
        r&   r   r   Nr   ^Speaker\s+(\d+)\s*:\s*(.*)$rM   r   r   z#No valid content found in text filer|   )r7   	readlinesr   rematch
IGNORECASEr   groupre   r@   r5   )
r   r   rD   linesr   current_speakerlinespeaker_matchru   rT   s
             r   r   z*VibeVoiceProcessor._convert_text_to_script  s    )S7333 	"qKKMME	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"  	J 	JD::<<D  H%DdBMZZM J !4!4Q!7!788
$**1--3355 I ''(G:(G(G(G(GHHH ##$H$H$H$$H$HIIII 	DBCCCyy&&&s   488r   c                    |                                                     d          }g }g }|D ]}|                                 st          j        d|                                 t          j                  }|ryt          |                    d                    }d|                    d                                           z   }|                    ||f           |                    |           t          	                    d| d           |st          d          t          |          }	|	d	k    r&g }
|D ]\  }}|
                    |dz
  |f            |
S |S )
z4Parse script into list of (speaker_id, text) tuples.r|   r   r    rM   zCould not parse line: ''z&No valid speaker lines found in scriptr   )r   splitr   r   r   r   r   re   r9   r:   r@   min)r   r   r   r   speaker_idsr  r   ru   rT   min_speaker_idnormalized_liness              r   r   z VibeVoiceProcessor._parse_scriptE  sz   $$T**  	B 	BD::<<  H<djjllBMZZE B Q00
U[[^^11333##Z$6777"":....@@@@AAAA 	GEFFF [))A!$0 @ @ 
D ''a(>????##  r    text_inputsaudio_inputsc                 `    t          |          }d|v r|d         |d<   d|v r|d         |d<   |S )z8Merge text and audio inputs into a single BatchEncoding.audior~   	streaming)r	   )r   r  r  mergeds       r   _merge_inputsz VibeVoiceProcessor._merge_inputsj  sL     {++ l""&27&;F?#,&&".{";F;r    c                 &     | j         j        |i |S )z
        This method forwards all its arguments to VibeVoiceTextTokenizer's [`~PreTrainedTokenizer.batch_decode`].
        Please refer to the docstring of this method for more information.
        )r   batch_decoder   argsr   s      r   r  zVibeVoiceProcessor.batch_decodew  s    
 +t~*D;F;;;r    c                 &     | j         j        |i |S )z
        This method forwards all its arguments to VibeVoiceTextTokenizer's [`~PreTrainedTokenizer.decode`].
        Please refer to the docstring of this method for more information.
        )r   decoder  s      r   r  zVibeVoiceProcessor.decode~  s    
 %t~$d5f555r    c                     | j         j        }| j        j        }t          t                              ||z   ddgz                       S )zB
        Return the list of inputs accepted by the model.
        r~   r   )r   model_input_namesr   r_   r   fromkeys)r   tokenizer_input_namesaudio_processor_input_namess      r   r  z$VibeVoiceProcessor.model_input_names  sQ    
 !% @&*&:&L#DMM"7:U"UYhj}X~"~  A  A  	Ar    
output.wavaudio_r  output_pathr)   	normalizebatch_prefixc                 @    | j                             |||||          S )a  
        Save audio data to a file.
        Args:
            audio (Union[torch.Tensor, np.ndarray, List[Union[torch.Tensor, np.ndarray]]]):
                The audio data to save. Can be a single tensor/array or a list of them.
            output_path (str, optional): Path to save the audio file. Defaults to "output.wav".
            sampling_rate (int, optional): Sampling rate for the audio. If None, uses the processor's default.
            normalize (bool, optional): Whether to normalize the audio before saving. Defaults to False.
            batch_prefix (str, optional): Prefix for batch audio files. Defaults to "audio_".
        Returns:
            str: The path to the saved audio file.
        )r  r)   r   r!  )r   
save_audio)r   r  r  r)   r   r!  s         r   r#  zVibeVoiceProcessor.save_audio  s6    & #..u+]jv  O[.  \  \  	\r    )NNr   T)NNTFNNTrr   )TFNNT)NNN)r  NFr  )-__name__
__module____qualname____doc__r   classmethodr?   r   r^   r0   PathLikerS   r   r   r   r   ra   rb   r   r
   r   r   r   r	   ro   r   r   rd   rf   r   r   r   r   r   r   r   r   r   r  r  r  propertyr  Tensorr#  rs   r    r   r   r      s        "_ _ _ _ D
 D
 [D
L GeC4D.E  G  G  G  GH y}ko59;@$(;?&*Q QuS$s)Y8I4PY?\`ar\sstuQ  d5bj+A&BDeTWY[YcTcNdIeDf&f ghQ tS/12	Q
 $%778Q SMQ !sJ!78Q  $Q 
Q Q Q Ql AE:
 :
CN#:
  U3
?%; <=:
 
c3h	:
 :
 :
 :
~ 6:;@$(;?&*b bS#X'b tS/12b $%778	b
 SMb !sJ!78b  $b 
b b b bH5JeCO455J 
tCy$rz*DJ6	75J 5J 5J 5Jt <@59'+0 0BJ'0 !sJ!780 sEL012	0
 $0 
c3h0 0 0 0d-' -' -' -' -' -'^$' $' $' $' $' $'L# C # DsCx,A #  #  #  # J d }    < < <6 6 6 A A XA ('+$\ \U\2:tE%,
:R4S/TTU\\  }\ 	\
 \ 
\ \ \ \ \ \r    r   )r   warningstypingr   r   r   r   r   r   r0   r   numpyra   r   $transformers.tokenization_utils_baser	   r
   r   r   r   transformers.utilsr   r   r2   r   
get_loggerr$  r9   r   __all__rs   r    r   <module>r3     sG     : : : : : : : : : : : : : : : : 				 				      B  B  B  B  B  B  B  B  B  B  B  B  B  B 2 2 2 2 2 2 2 2 : : : : : :		H	%	%P
\ P
\ P
\ P
\ P
\ P
\ P
\ P
\f r    