o ॵin@sjddlZddlmZeZdZddZddZdd Zd d Zd d Z ddZ ddZ ddZ ddZ dS)N) get_loggeruT[’!"#$%&'()*+,-./:;<>=?@，。?★、…【】《》？“”‘’！[\]^_`{|}~]+cCsrg}|}t|dkr7td|}|dur|d}n|dd}||||ddd}t|dks |S)Nrz[A-Za-z!?,<>()\']+ )lowerlenrematchgroupappendreplacestrip) input_strtokenssr wordrb/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/audio/kws_utils/file_utils.pysplit_mixed_labels      rcCs$t|}ddd|D}|S)Nrcss|]}|dVqdS)rNr).0subrrr (sz$space_mixed_label..)rjoinr )rsplits space_strrrrspace_mixed_label&srcCs\g}t|ddd}|D]}|dkr||q Wd|S1s'wY|S)Nrutf8encodingr)openr r ) list_filelistsfinlinerrr read_lists,s  r%c Csi}|D]+}|dd}t|dkr!td|q||dd||d<qg}|D]6}|dd}t|dkr`|d|vr`|t|d||d|ddd q4td |dq4|S) N rzinvalid line in trans file: {}rrri>)keytxtwav sample_ratez*can't find corresponding trans for key: {}) r r splitrloggerdebugformatr dict) wav_lists trans_lists trans_tabler$arrr"rrr make_pair5s0  r5cCs|i}t|ddd'}|D]}|}t|dksJt|dd||d<q Wdn1s3wY||S)Nrrrr'rr)r r r,rintclose) token_file tokens_tabler#r$r4rrr read_tokenQs r:cCsi}t|ddd)}|D]}|dd}t|dks J|dd||d<q Wdn1s5wY||S) Nrrrr&rr'rr)r r r r,rr7) lexicon_file lexicon_tabler#r$r4rrr read_lexicon\sr=cCst}t}t|}|D]a}|dks|dks|dkr|d}q |dks'|dkr,|d}q |dks<|d ks<|d ks<|d krA|d }q ||vrK||f}q ||vr\||D]}||f}qSq ttd |}|D]}||f}qeq |D]c}||vr~|||f}qp|dkrd|vr||df}qp||df}qp|dkrd|vr||df}qp||df}qpd|vr||df}td|dqp||df}td|dqp||fS)N!sil(sil))r>)rA(noise)noise)(noise)rsilrG')' is not in token set, replace with )' is not in token set, replace with )tuplerrr symbol_strr-infor) symbol_tabler< tokens_str tokens_idxpartspartchrrrquery_token_setgsR          rVcCsg}g}t|}|D]d}|dks|dks|dkr|dq |dks&|dkr,|dq |dks<|dks<|dks<|d krB|d q ||vrL||q ||vr]||D]}||qTq ttd |}|D]}||qfq |D]c}||vr|||qq|dkrd |vr||d qq||dqq|d krd |vr||d qq||dqqd |vr||d td |dqq||dtd |dqq||fS)Nr>r?r@rArBrCrDrErFrGrrHrIrJrK)rr rrrMr-rNrOrrrquery_token_listsR          rWcCsF|D]}d|vs J|d}t|||\}}||d<||d<q|S)Nr)r)r rW) data_listrPr<sampler)strsindexsrrrtokenizes   r\)rmodelscope.utils.loggerrr-rMrrr%r5r:r=rVrWr\rrrrs    1 1