o
i | @ s d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl Z d dl
m Z d dl m
Z
d dlmZ d dlmZmZ d dlmZ d dlmZmZ d dlmZmZ d d lm Z d d
l m Z e j e j e j!d Z"e#d d
G dd
d
e j$Z%dS ) N)Union)compute_accuracy)tables)force_gatherable to_device)
DatadirWriter)
extract_fbankload_audio_text_image_video)
AutoConfigAutoModelForCausalLM)CTC) forced_align)bf16fp16fp32
model_classes
FunASRNanoc sR e Zd Z d4dededededed ed
ed ef fd d
Z d5dej dej dej dej dej dej dej fddZ
dd Z dd Z dd Z
i fd efd d Z d6d efd!d"Zd7d$ee d%ed&efd'd(Zd)ed*eeej f fd+d,Z d6d efd-d.Z d6d efd/d0Zed8d1efd2d3Z ZS )9r NP F
audio_encoderaudio_encoder_conf
audio_adaptoraudio_adaptor_confllmllm_conf
input_sizelength_normalized_lossc ! s t |dd }
|dd| _|
dkr>ddlm} | |dd} t| jd r,| jjnd
}
t| jd r9| jjj n| jj }nt
j |}|d-d |i|}| }
|d
d}|ri|
D ]\}}d|_q]| || _d | _|dd }d }|di }t|}tj|fi |} |d
d}|r|
D ]\}}d|_q| |ddr| |dd| _| t| j | _| jj d
}t
j |}|
dkr|
|d< |d ur|n|d |d< |d-i |}|d
d}|r|
D ]\}}d|_q| || _ |dd| _ d | _ t
j | dd }|d urd| v r%| dd n| d d }d| v r6| dd n| d d }|d urV|d urVt
j!|}|d-i |}|| _"|d us_J d| dd }| d i } |
dkrt|
| d< |d-i | | _ | dd }|d urt#j$|d d } | j j%| dd } t&'d!| d"| | d
d}|r| j
D ]\}}d|_q| j | d#i } | d$|d% | _(| d&d'| _)t*d-||
| j(d(| | _+| d)d| _,d | _-|| _.t/t0j1d*d} t&'d+| d, d S ).Nhubactivation_checkpointFmsr AutoModelmaster)modelmodel_revisionencoder_output_sizer" r freezeTinit_param_path load_kwargs llm_dtyper encoder_dimllm_dimuse_low_frame_rate ctc_decoder
ctc_tokenizer dataset_confctc_tokenizer_confzctc_tokenizer must be setctc_vocab_sizeic ctc_decoder_confcpu) map_location)strictzLoading ctc_decoder ckpt: z
, status: ctc_confblank_id
ctc_weightg333333?)odimr$ r7 detach_ctc_decoderRANKzrank: z, model is builded. )2super__init__get#audio_encoder_activation_checkpointfunasrr hasattrr" r$ encoderr encoder_classes output_sizenamed_parameters
requires_gradevalr r r
from_pretrainedr from_config gradient_checkpointing_enabler) to dtype_mapget_input_embeddingsweightshapeadaptor_classesr r, r- tokenizer_classesr. torchloadload_state_dictlogginginfor7 r9 r ctcr; error_calculatorr intosenviron)!selfr r r r r r r r kwargsr r r" audio_encoder_output_size
encoder_classr& _paramr' r+ llm_load_kwargsconfig
adaptor_classctc_decoder_classr. r0 ctc_tokenizer_classr1 r2 src_stateflagr6 rank __class__r= T/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/fun_asr_nano/model.pyr? s
zFunASRNano.__init__speechspeech_lengths input_idsattention_mask
labels_ids fbank_beg
fbank_maskc % K s( |j \} }
i } d||dk < | jj |} |d ur+t| dkr*|d d df }|j \}
}}| jrEddlm} || j ||dd\}}n| ||\}}|
||\}}| j \} }
}| d}d||dk < d||dk < d}t | D ]}t |j d D ]}|||f
}|dkr
|||f }||d |d d f }z|| |||| d d f< W n^ ty } zQtt| dt td| d | j d
| d | d |j d
| d| d| ||
}||d |d d f }|| |||| d d f< W Y d }~nd }~ww |d7 }q{qr|
| d< ||
| d< |
| d< | d | d | d< t| jj} tj| dv r<| nd| j dkrEdndt | j d% d||dk< d||dk < | j| t | j ||d} | j } W d n 1 svw Y t * t!| j"d} t#| d d d df |d d dd f dd } | | d < W d n 1 sw Y t$| % | d < | | d < |
| | d < |
| d!< | d | d! | d"< |dkd}!t&|!'
}"|!
| }#|"| d#< |#| d$< | j(rt'|dk } t)| | | f| j\} } }$| | |$fS )%Nr r8 )
checkpointF)
use_reentrantfake_token_len, batch_idx: , inputs_embeds: , fbank_beg_idx: , speech_token_len: z, encoder_out: z, encoder_out_lens: , fake_token_len: , speech_lengths: batch_size_speechbatch_size_x_framesbatch_size_real_framespadding_framescudaxpumpsr3 r T device_typeenableddtyper% )
inputs_embedsrr labels) ignore_labelaccloss
batch_sizebatch_size_x_tokensbatch_size_real_tokenspadding_tokensdialog_turns_maxdialog_turns_avg)*rQ r r" rO lensizerA torch.utils.checkpointrv encoder r@ rangeitem ExceptionrW errorstr traceback
format_excrX sumnext
parametersdevicetyperT autocastr) rN rM r no_gradargmaxlogitsr clonedetachmaxr[ r r )%r^ ro rp rq rr rs rt ru r_ r token_numstatsr r framesrb rv encoder_outencoder_out_lensdimsrx
speech_idx batch_idxturn_id
fbank_beg_idxspeech_token_len speech_tokener
model_outputsr predsacc_att dialog_turnsr r rP r= r= rn forward s
6
.
zFunASRNano.forwardc K s( | ||\}}| ||\}}||fS N)r r )r^ ro rp r_ xolensr r r= r= rn forward_export s zFunASRNano.forward_exportc C s | ||\}}||fS r )r )r^ ro rp r r r= r= rn r s zFunASRNano.encodec C s g g g }}}t |D ]5\}}|d }|d }|dkr"|| q |dkr8d|v r2|d } || g}|| q |dkrA|| q |t| }|||d}
|
S )Nrolecontentsystemuseraudio assistant)r r r ) enumerateappendr ) r^ datar r r ir r r r contentsr= r= rn
data_template s(
zFunASRNano.data_templater c /
K s< |d }|d }|d }t d} d}
d} d|v r)|d dd}
|d dd} g g g g g g g f\} }
}}}}}g }tt|||D ]\}\}}}||d d
krV nt| |d d krc nt|ttfrn|\}}|d
kr|ddrd| d| }| sd| }n$d| d| d}| sd| d}n|ddrd| }nd| d}|
s|d7 }|dd d ur||d 7 }| |}g }g } d
} d} g g } } t|D ]\}!}"|"
ds| |"}#||#7 }| d
gt|# 7 } q|" dd dd}"|"
dr|"dd }"|"
dr|}"z t
}$t|"fd |ji|}%t
}&|&|$ d |d < W n" tyO }' ztd t|' d t W Y d }'~'nd }'~'ww t|%|d!d"|dd#\} } t
}(|(|& d |d$< | |j |j d% |d&< | j rd| d
d' d( d( })d|)d' d( d( })|)d d( d } n| d
} d
g| }*t|} ||*7 }| dgt|* 7 } q|| t| g7 }|| g7 }d)gt| }+| d*}| |},| | }| ||, 7 } |
|+|, 7 }
|| 7 }t| d
kr | | d
d d d d f | | qCt j | t j d+} t j dgt| t j!d+}-t j |
t j d+}
t j |t j"d+}t j |t j!d+}t j |t j!d+}t j |t j d+}t j |,t j d+},t|d
krmt j#j$j%j&|dd,d-} t j#j$j%j&|ddd-} ng } g } | | |d d d f |d. |d d d f | d. |-d. |
|d d d f |,d d d f d/
}.|.S )0Nr r r z)(<\|startofspeech\|>.*?<\|endofspeech\|>)Tr/ do_think
sys_promptmultiturn_num_max max_token_lengthi r infer_with_assistant_inputFz<|im_start|>system
z <|im_end|>
<|im_start|>user
z<|im_start|>user
z!<|im_end|>
<|im_start|>assistant
z
prev_textr% z<|startofspeech|> <|endofspeech|>!r8 fsz0.3f load_datazLoading wav failed! ry data_typesound)r frontendis_final extract_feat batch_data_time r z
<|im_end|>r g ) batch_first
padding_valuer )
ro rp ru rt rx rq rr rs
source_ids
target_ids)'recompiler@ r zipr
isinstancelisttuplesplit
startswithr replacetime perf_counterr r r rW r r r r r r r frame_shiftlfr_nr, r rT tensorint64int32float32nnutilsrnn pad_sequence)/r^ r tokenizerr meta_datar_ r r r patternr r rq r fbank
fbank_lensru rt rx input_source_idsr
system_prompt user_prompt
target_outr source_inputsplitsr fbank_mask_ifake_token_len_i fbank_beg_iro rp ksub_str sub_tokentime1data_srctime2r time3r
fake_token source_maskr rr outputr= r= rn data_load_speech4 s
*
zFunASRNano.data_load_speechkeyc K s i }| dddkrtd| |d }| j|||fd|i|} t| |d }
|
d } t| dkrd|v rDd |v rD|d } |d }
nC|
d
d d df }| d d r[| tj} n | d
d rg| tj } |
| |\} }
| | |
\}}| |d< |
|d< ||d< ||d< |
d }|
d }|
d }|
d }| dd s|}d||dk < | j j
|}|j\}}}d||dk < d||dk < d}t|D ]}t|jd D ]}|||f } | dkr`|||f } ||d | d d f } z| ||| | | d d f< W n^ ty[ } zQtt| dt td| d|j d| d| d |j d | d | d | || } ||d | d d f } | ||| | | d d f< W Y d } ~ nd } ~ ww |d7 }qq|||
||fS ) Nr r8 z!batch decoding is not implementedr r r ro audio_embeddingaudio_embedding_lensrp r Fr r r audio_adaptor_outaudio_adaptor_out_lensrq r rt rx teacherforcingry rz r{ r| r} z, adaptor_out: z, adaptor_out_lens: r~ r )r@ NotImplementedErrorr r r r rM rT float16bfloat16r r r r" rO rQ r r r rW r r r r rX ) r^ data_in data_lengthsr r r r_ r r r batchro r r rp adaptor_outadaptor_out_lensrq r rt rx r r r r r r r r r r r r= r= rn inference_prepare s
6
z FunASRNano.inference_prepareThotwordslanguageitnc C sd t |dkrd|}d}|d| d7 }nd}|d u r!|d7 }n|d| 7 }|s.|d 7 }|d
S ) Nr ry u 请结合上下文信息，更加准确地完成语音转写任务。如果没有相关信息，我们会留空。
**上下文信息：**
u 热词列表：[z]
r u 语音转写u 语音转写成u ，不进行文本规整u ：)r join)r^ r( r) r* promptr= r= rn
get_prompt& s
zFunASRNano.get_promptr, r c C sf t |trdddd| d| dddddgS t |tjr1dddd| d |d
dddgS d S ) Nr z You are a helpful assistant.)r r r z<|startofspeech|>!r r nullz"<|startofspeech|>!!<|endofspeech|>)r r r )r r rT Tensor)r^ r, r r= r= rn generate_chatml5 s
zFunASRNano.generate_chatmlc s |dg |dd |ddfdd|D }|d u rAg }|D ]}tjtj |dd fd d
td D q%j|f||||d |S )
Nr( r) r* Tc s g | ]} |qS r= )r0 ).0r )r, r^ r= rn
S s z(FunASRNano.inference.. rand_key_r c 3 s | ]}t V qd S r )randomchoice)r1 rb )charsr= rn Y s z'FunASRNano.inference..
)r# r r r ) r- r@ string
ascii_lettersdigitsr r+ r
inference_llm)r^ r" r# r r r r_ rb r= )r6 r, r^ rn inferenceG s& (zFunASRNano.inferencec / K s | j |||||fi |\}}} }
} g } | jd ur| d }
| d }| |
|\}}| j|}|
\}}}t|d ttfrC|d }t||k rM|| }t |D ]:}||d ||
d d f }|j dd}t j
|dd}|| jk}|| }| j|}| || ||d qQ|dd}|dkr|d d
rd n|}|d d
rd n|}t |d d
j} t j| dv r| nd|dkrdnd
t| d |d d } | jt| | _|t| }|di } |dd
s | dd } | jjd;|| |dd| jjj p | jjj d| } |j | |dddd }!d }"nK| d }#d |#|#dk< | dd } | jd;|| |#| jjj p?| jjj d | }$t |$j dd d |
j d d f }%|j |%d
|ddd d }!|$j!
}"W d n 1 suw Y |d d!|! }!d }&|d"d urt"| d#st#|d"| _$| j$d d$ }&g }'t%&d%d!|!}(|d t%&d&d'|!'d(d'|(| d)})|"d ur|"|)d*< |'|) t(| |'D ]v\}*}+|*d+ 'd,d!|+d-< t j)| j*|+d- t j+d.},t,|*d/ |,| j|+d0< t j)| j*|+d+ t j+d.},t,|*d/ |,| j|+d1< |+d1 |+d0 fD ],}-|-D ]&}.| j|.d2 g|.d2< |.d3 d4 d5 d6 |.d3< |.d7 d4 d5 d6 |.d7< q qq|&d url|!'d8d'|&d+ |d < | 'd8d'|&d9 |d < |(|&d: |d < |'| fS )ctc_textr r@ ctc_timestamps
timestampstoken
start_time
r end_time
rL rK r= )-r' r- rY log_softmaxr r r r r r r r rT unique_consecutiver7 tolistr. decoder r@ r r r rN r rM generatere rD eos_token_id batch_decoder rQ r rC r rH r subr r r r r r
)/r^ r" r# r r r r_ r r r$ r r ctc_resultsr r decoder_outdecoder_out_lensr@ bndr r yseqmask token_intr? r) r rL rA rr
generated_idsresponser rs r r ibest_writerresultsresponse_cleanresult_i
ctc_resultresultr rO timestampr= r= rn r< d s
$
,
zFunASRNano.inference_llmr" c K s. ddl m} |jd| dd|\} }| |fS )Nr r T)r" trust_remote_coder= )rB r build_model)r" r_ r r= r= rn rJ s zFunASRNano.from_pretrained)NNNNNNr F)NNNNNNN)NNNN)NTr )__name__
__module__ __qualname__r dictr[ boolr? rT r/ r r r r r r r' r- r r0 r= r< staticmethodrJ
__classcell__r= r= rl rn r s
o
Y
)&rW r\ r4 r r9 r r typingr rT torch.nnr funasr.metrics.compute_accr funasr.registerr funasr.train_utils.device_funcsr r funasr.utils.datadir_writerr funasr.utils.load_utilsr r transformersr
r rY r tools.utilsr
r! r r rN registerModuler r= r= r= rn s*