o ui6@sddlZddlmZmZmZmZddlZddlZddl Z ddl m Z m Z m Z mZddlmZmZddlmZmZmZddlZGdddeZ       dd ed edededeededeededeededefddZdS)N)ListUnionOptional NamedTuple) N_SAMPLES SAMPLE_RATE load_audiolog_mel_spectrogram)TranscriptionResult SingleSegment) WhisperModelFasterWhisperPipelinefind_numeral_symbol_tokensc seZdZdZ     ddedeeed fd effd d Z d e j fddZ       dd eee j fde edefddZ  dde ed eee j fdedededef ddZZS)VadFreeFasterWhisperPipelinez( FasterWhisperModel without VAD NptenFoptionsdevicez torch.devicesuppress_numeralsc s*tjd|di||||||d |dS)a Initialize the VadFreeFasterWhisperPipeline. Args: model: The Whisper model instance. options: Transcription options. tokenizer: The tokenizer instance. device: Device to run the model on. framework: The framework to use ('pt' for PyTorch). language: The language for transcription. suppress_numerals: Whether to suppress numeral tokens. **kwargs: Additional keyword arguments. Returns: None N) modelvad vad_paramsr tokenizerr frameworklanguager)super__init__) selfrrrrrrrkwargs __class__r=/home/ubuntu/sommelier/podcast-pipeline/models/whisper_asr.pyrs z%VadFreeFasterWhisperPipeline.__init__audioc Cs|jjd}|jdtkr#tjd|jdt}|||t}n|dt}|jdtkr2dnt|jd}t||durA|nd|d}|j |}|jj |}|dd\} } | dd} | | fS)z Detect the language of the audio. Args: audio (np.ndarray): The input audio signal. Returns: tuple: Detected language and its probability. feature_sizerNP)n_melspadding) r feat_kwargsgetshapernprandomrandintr encodedetect_language) rr$ model_n_mels start_index audio_sampler(segmentencoder_outputresultslanguage_tokenlanguage_probabilityrrrr#r2>s      z,VadFreeFasterWhisperPipeline.detect_languager vad_segmentsreturnc  CsRt|tr t|}dd} |dur"||} t| tr | d}n| }t|tr+|d}ddg} |durA|| vrAtd|dd}|jdur[|pId}tjj|j j |j j j ||d |_n?|p`|jj }t |jd rt|jj| vrt|pr|jj}n|pwd}|| vr~d}||jjks||jj krtjj|j j |j j j ||d |_|jr|jj} t|j}||jj}tt|}|jj|d |_g}|p|j}t|}tj|d d }t|j| ||||dD]5\}}|r|d|d}|dvr|d}||t||ddt||dd||dddq|jdurd|_|jr$|jj| d |_||dS)a Transcribe the audio into text. Args: audio (Union[str, np.ndarray]): The input audio signal or path to audio file. vad_segments (List[dict]): List of VAD segments. batch_size (int, optional): Batch size for transcription. Defaults to None. num_workers (int, optional): Number of workers for loading data. Defaults to 0. language (str, optional): Language for transcription. Defaults to None. task (str, optional): Task type ('transcribe' or 'translate'). Defaults to None. chunk_size (int, optional): Size of chunks for processing. Defaults to 30. print_progress (bool, optional): Whether to print progress. Defaults to False. combined_progress (bool, optional): Whether to combine progress. Defaults to False. Returns: TranscriptionResult: The transcription result containing segments and language. cssB|D]}t|dt}t|dt}d|||iVqdS)Nstartendinputs)intr)r$segmentssegf1f2rrr#data{s z5VadFreeFasterWhisperPipeline.transcribe..dataNr transcribe translatezWarning: Invalid task 'z', defaulting to 'transcribe'taskrrJ)suppress_tokens Transcribing)totaldesc) batch_size num_workerstext)rrQNr>r?speaker)rRr>r?rT)rBr) isinstancestrrr2tupleprintrfaster_whisper Tokenizerr hf_tokenizeris_multilingual language_codehasattrrJrrrKrlistset_replace _batch_sizelentqdm enumerate__call__updateappendroundr,preset_language)rr$r<rOrPrrJ chunk_sizeprint_progresscombined_progressrFdetected_language valid_tasksprevious_suppress_tokensnumeral_symbol_tokensnew_suppressed_tokensrBtotal_segmentsprogressidxoutrRrrr#rG[s               z'VadFreeFasterWhisperPipeline.transcribecudarBrreturn_char_alignmentsc Cst|tr t|}n|}z tj||d\}}Wn%ty<} ztd|d| td||dWYd} ~ Sd} ~ wwztj||||||d} | WStym} ztd| td||dWYd} ~ Sd} ~ ww) au Align transcribed segments to get word-level timestamps using WhisperX alignment. Args: segments (List[dict]): Transcribed segments from transcribe(). audio (Union[str, np.ndarray]): The input audio signal or path to audio file. language (str): Language code for the alignment model. device (str, optional): Device to run alignment on. Defaults to "cuda". return_char_alignments (bool, optional): Whether to return character-level alignments. Defaults to False. Returns: dict: Aligned segments with word-level timestamps. )r]rz6Warning: Could not load alignment model for language 'z': z0Returning segments without word-level alignment.)rB word_segmentsN)rxzWarning: Alignment failed: )rUrVrwhisperxload_align_model ExceptionrXalign) rrBr$rrrx audio_arraymodel_ametadataealigned_resultrrr#align_segmentss<  z+VadFreeFasterWhisperPipeline.align_segments)NrrrF)NrNNr;FF)rwF)__name__ __module__ __qualname____doc__rrrArVboolrr.ndarrayr2rdictr rGr __classcell__rrr!r#rsZ (!   rfloat16rrG whisper_archr device_index compute_type asr_optionsrrrJ download_rootthreadsr=c  Cs$|drd}|pt||||| | d}|dur%tjj|j|jj| |d} ntdd} iddd dd d d d d d dddgdddddddddddddddddd d!gd"dd#dd$d%ddddddd& } |dury| || d'}| d'=tj j d)i| } t || | ||d(S)*a Load a Whisper model for inference. Args: whisper_arch (str): The name of the Whisper model to load. device (str): The device to load the model on. device_index (int, optional): The device index. Defaults to 0. compute_type (str, optional): The compute type to use for the model. Defaults to "float16". asr_options (Optional[dict], optional): Options for ASR. Defaults to None. language (Optional[str], optional): The language of the model. Defaults to None. vad_model: The VAD model instance. Defaults to None. vad_options: Options for VAD. Defaults to None. model (Optional[WhisperModel], optional): The WhisperModel instance to use. Defaults to None. task (str, optional): The task type ('transcribe' or 'translate'). Defaults to "transcribe". download_root (Optional[str], optional): The root directory to download the model to. Defaults to None. threads (int, optional): The number of CPU threads to use per worker. Defaults to 4. Returns: VadFreeFasterWhisperPipeline: The loaded Whisper pipeline. Raises: ValueError: If the whisper architecture is not recognized. z.enr)rrrr cpu_threadsNrIz`No language specified, language will be detected for each audio file (increases inference time). beam_sizebest_ofpatiencerQlength_penaltyrepetition_penaltyno_repeat_ngram_sizer temperatures)g?g?333333?g?g?compression_ratio_thresholdg333333@log_prob_thresholdgno_speech_thresholdrcondition_on_previous_textFprompt_reset_on_temperatureg?initial_promptprefixsuppress_blankTrKrwithout_timestampsru "'“¿([{-u"'.。,，!！?？:：”)]}、) max_initial_timestampword_timestampsprepend_punctuationsappend_punctuationsrmax_new_tokensclip_timestampshallucination_silence_threshold multilingualhotwordsr)rrrrrr) endswithr rYrrZr[rr\rXrgrGTranscriptionOptionsr)rrrrrr vad_model vad_optionsrrJrrrdefault_asr_optionsrrrr#load_asr_models &       r) rrNrNNNrGNr)rYtypingrrrrtorchnumpyr.rdwhisperx.audiorrrr whisperx.typesr r whisperx.asrr r rrzrrVrArrrrrr#sZ