o
iW @ s d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlm Z d dl
m Z m Z e
ddG dd dejjjZe
ddG dd dejjjZdS ) N)tables)
extract_fbankload_audio_text_image_videodataset_classesFunASRc sr e Zd ZdZ ddededef fdd
Zd
d Zd d
Z dd Z
dd Z dd Z dde
fddZ ZS )r z
FunASR dataset
N index_ds
int_pad_valuefloat_pad_valuec s t tj|}||fi || _|dd } | r,tj| }
|
d(i |d} | | _|dd } | rGtj| } | d(i |d} | | _|dd }
|
d uretj |
}|d(i |d}nd }|| _ || _
|d u rsdn|j | _ d| _ || _
|| _|| _|d d
| _|d d | _|d
| _|d| _d| _|dd| _td| _|dd| _|dd| _|dd| _|dd| _ |dd | _ |d d | _ |d d | _ |d!d | _ |d"d#| _!|d$d%| _"|d&d'| _#d S ))Npreprocessor_speechpreprocessor_speech_confpreprocessor_noisepreprocessor_noise_confprompt_classes prompt_confi> soundsosz<|startoftranscript|>eosz
<|endoftext|>
batch_size
batch_typer retryd z)(<\|startofspeech\|>.*?<\|endofspeech\|>)max_token_lengthi batch_size_scale_ratio_maxg ?batch_size_token_maxi multiturn_num_max max_source_lengthi max_target_lengthi do_thinkT
sys_promptuse_dynamic_output_ratior min_mask_token_len min_non_mask_token_len )$super__init__r index_ds_classesgetr preprocessor_classesr r r frontendfs data_type tokenizerr
r r r r r prompt_ids_lenr recompilepatternr r r r r r r r! r" min_output_mask_token_len min_output_non_mask_token_len)selfpathr r- r0 r
r kwargsindex_ds_classr preprocessor_speech_classr preprocessor_noise_classprompt_classes_textr __class__r' ]/home/ubuntu/.local/lib/python3.10/site-packages/funasr/datasets/fun_asr_datasets/datasets.pyr) sZ
FunASR.__init__c C | j | }| j |S N)r get_source_lenr7 indexitemr' r' r@ rD T
FunASR.get_source_lenc C rB rC )r get_target_lenrE r' r' r@ rJ X rH FunASR.get_target_lenc C s ddg}| dd }d }|d ur1| dkr!|d |d n| dkr1|d |d t|d
kr=t|}nt|d krNtj|g d d
d }d|v r\|d|d }|S d|v rh|d|d }|S )Nu 语音转写：zSpeech transcription:languagezhu 语音转写成中文：z Transcribe speech into Chinese:enu 语音转写成英文：z Transcribe speech into English: )皙?rQ 皙?rR )weightsr u 语音转写：<|startofspeech|><|startofspeech|>z&Speech transcription:<|startofspeech|>)r+ lowerappendlenrandomchoicechoicesreplace)r7 rG user_prompttasksrL taskr' r' r@ get_random_user_prompt\ s(
z FunASR.get_random_user_promptc C
t | jS rC )rW r r7 r' r' r@ __len__r
FunASR.__len__c 1 C s6 d }t | jD ]}|dkrtd| d}|dkr |}n tdt| jd }| j| }|d }|d }|d } | dd}
t|d k sNt| d k rWt
d
| qg g g g g g g f\} } }
}}}}t t ||| D ]?\}\}}}|| j
kr n1t| | jkrtd t| d | j d
| n| jd ur|dd }| j|}| j||}nd}|dkrd| d| | d}| jsd| | d}nd| d}| js|d7 }| j|}g }g } d} d} g } g } g }!t |D ]\}"}#|#ds | j|#}$||$7 }q|#dddd}#|#drz^t|#d d | jd}%| j d urU|
sUz | |% }%W n t yT }& zt d|& W Y d }&~&nd }&~&ww t |%| j!| j"dd\} }!|!| j#krxtd |! d | j# d
| d}W n' t y }& zt
d t$|& d
t%& d | d}W Y d }&~&qd }&~&ww d |!d d d d }'d |'d d d }'|'d d d } dg| }(t|} ||(7 }q|rqo| dkr|| t| g7 }|| g7 }n
|dg7 }|dg7 }|d ur"t'd!d" |D r"t(d#d" |D d })|| j|)7 }d$gt| }*|d d%}n d$gt| }*| d%}| j|}+t|+| j)krMtd&t|+ d'| j) d(| t*j+, | j-k rt|+},t.| j/|,}-t.| j0|,}.|,|. |-krwt*j+|-|,|. }/n|,|. }/|/dkrd$g|/ |+d |/< | ||+ 7 } | |*|+ 7 } t| dkr|
1| dd d d d f |1|! qo|rqtj2| tj3d)} tj2d gt| tj4d)}0tj2| tj3d)} tj2|tj4d)}tj2|tj4d)}||| |0| d*}||d+< t|
dkr|
|d,< ||d-< t| | jkrt
d.t| d/| j d0| q |S |S )1Nr zretry: Fr' systemuser assistantnoisedr$ zitem is error: z input_ids > max_token_length: >, rT z<|im_start|>system
z <|im_end|>
<|im_start|>user
z!<|im_end|>
<|im_start|>assistant
z<|im_start|>user
z
r z<|endofspeech|>!)r. z Generate noise audio failed: T)r/ r- is_finalz$speech_lengths > max_source_length: zLoading wav failed!
rO c s s" | ] }t |to d |v V qdS prev_contentN
isinstancedict.0rG r' r' r@ s
z%FunASR.__getitem__..c s s* | ]}t |trd |v r|d V qdS rp rr ru r' r' r@ rw s iz
<|im_end|>z
text_length: z > z , drop it: )dtype) fbank_begfake_token_len input_idsattention_mask
labels_idsrG speechspeech_lengthszlen(input_ids): z > max_token_length: z, item: )5ranger logginginfotorchrandintrW r rG r+ warning enumeratezipr r r splitdetect_language
get_promptr! r r4
startswithr0 encoder[ r r. r numpy Exceptionerrorr r/ r- r str traceback
format_excanynextr nprX randr" minr5 r6 rV tensorint64int32)1r7 rF outputidx badcase_flag index_currG re rf rg is_noisedr{ labelsfbank
fbank_lens
fbank_maskry rz i
system_promptr\
target_out
asr_promptrL user_prompt_all_context source_inputsplits
source_idsfbank_ifake_token_len_i fbank_beg_i fbank_lens_ir~ r ksub_str sub_tokendata_srceolens
fake_token
prev_value source_mask
target_idsmax_lenr5 r6 end_indexr| r' r' r@ __getitem__u sP
FunASR.__getitem__samplesc C s` t | jD ]}d}i }|D ]2}|d u rq
| D ]&}||vr"g ||< t|| ttfr5|| || q|| || qq
| D ]0\}}t|d t j
rt|d j t j ks`|d j t j
krd| j}n| j}t jjjj|d|d||< qD| jdkr|d j\} }
| dkr| |
| jkrtd| d | d
|
d | |
d | j d
|d d }q |S |S )NFr T) batch_first
padding_valueexampler{ r$ z Warning, z th, b*t: *=z > batch_size_sample_max: z, drop last datar )r r keysrs listtupleextendrV itemsr Tensorrx r r r
r nnutilsrnn pad_sequencer shaper r r ) r7 r r r outputssamplekey data_list pad_valuebtr' r' r@ collator5 s@
( zFunASR.collator)NNNr r rC )__name__
__module__ __qualname____doc__r intfloatr) rD rJ r_ rb r r r
__classcell__r' r' r> r@ r
s* A Ar* c sB e Zd Zdef fdd Zdd Zdd Zdd Zd
d Z Z S ) r r8 c sX t |dd| _|dd| _|dd| _|dd| _t|d d
}|d d }|d
s|ds|dd}|dd}|sKd}d}t |dd6}|
}t |d | d } ||| |d | }
t
d| d| d| d|
d|
W d n1 sw Y n|g}
g } d} d}
|
D ]}t | dd}|D ]}z t| }W n ty } zt d| d| d | W Y d }~qd }~ww |d }t|d dttfrt|d dgd }t|d dd }nt|d d}t|d d}t|}t|}|dkr |dk r q|dk rEt d | d!| d"| d | t |d#krDt |d# d$ }q|| jkrLq|| jk rSq|| jkrZqg g g }}}t|D ]\\}}z
|d% }|d$ }W n ty t d&| d | Y qfw |d'kr|| qf|d(kr|| qf|d)krd*|v r|d* } ||d*| ig qf|| qft |dkrd+g}|t | }||||| d,} d-|v rt|d-d.ttfs|d- n|d- d | d-< d/|v r|d/ | d/< d0|v r|d0 | d0< d1|v r|d1 | d1< d2|v r%|d2 | d2< d3|v r0|d3 | d3< d4|v r;|d4 | d4< d5|v rF|d5 | d5< d6|v rQ|d6 | d6< |d7d8r\|| d9< | |d: d; d< | 7 } |
||d
d= d= d= 7 }
| | qW d n 1 sw Y q| | _t
d>t | j d?| d@dA|
dBdC| dC|
dD d S )ENr i@ min_source_length
r i min_target_lengthr audio_downsample_rate is_trainingTz.jsonlz.jsondata_split_numr$ data_split_izutf-8)encodingz
is_training: z, data_split_num: z, data_split_i: z,
file_list: z,
file_list_all: r zdrop it, json error: z, line: z
, file_json: messages
speech_length text_lengthzspeech_length: z, text_length: z, data: rO contentrolezdrop it, KeyError: re rf rg rq z You are a helpful assistant.)re rf rg
source_lenr key_01234 hist_contexthotwords asr_hotwordsvad_segs word_listone_pass_result one_pass_werrh save_metaFmetag Y@i i' i z
total_num of samplers: z, total_whrs: z.5fz, total_token_for_llm_B: z.5grj z
)r( r) r+ r r r r r endswithopen readlinesrW r r stripjsonloadsr r rs r r r r KeyErrorrV contents) r7 r8 r9 r r r r fin
file_list_all
num_per_slice file_listr
total_whrstotal_token_for_llm_B file_jsonline data_dictr datar r re rf rg r rG r r rq
contents_ir> r' r@ r) c s
^,rA c C r` rC )rW r ra r' r' r@ rb rc rd c C s | j | }|S rC )r )r7 rF r r' r' r@ r s
r c C s0 | dd}|dk rt|d t|d }|S )Nr r r re rf )r+ rW )r7 r r r' r' r@ rD s rI c C s dS )Nr r' )r7 r r' r' r@ rJ s rK )
r r r r r) rb r rD rJ r r' r' r> r@ r ` s )r r r2 r rX r r r funasr.registerr funasr.utils.load_utilsr r registerr r Datasetr r' r' r' r@ s
T