o
it @ s. d dl mZ d dlmZ d dlmZmZ d dlZd dlZd dl Z d dl
m m Z
d dl mZ d dl m Z d dlZd dlmZmZ d dlmZ ed d
ed d ed d ed d
ed dG dd de jZed ded ded ded ded dG dd de jZdS ) ) dataclass)Dict)IterableOptionalN)Tensor)nn)load_audio_text_image_video
extract_fbank)tables
model_classeszQwen/Qwen-Audioz
Qwen-AudiozQwen/QwenAudio QwenAudio
QwenAudioWarpc s@ e Zd ZdZ fddZdd Z d
defdd Z ZS ) r
z
Qwen-Audio: Advancing Universal Audio Understanding via Unified Large-Scale Audio-Language Models
https://arxiv.org/abs/2311.07919
Modified from https://github.com/QwenLM/Qwen-Audio
c s` t ddlm}m} ddlm} |dd}|j|ddd}|j|dd }|| _ || _
d S )
Nr AutoModelForCausalLM
AutoTokenizerGenerationConfig
model_pathr cpuT)
device_maptrust_remote_coder super__init__ transformersr r transformers.generationr getfrom_pretrainedmodel tokenizer) selfargskwargsr r r
model_or_pathr r __class__ R/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/qwen_audio/model.pyr s
zQwenAudioWarp.__init__c C d S Nr' r! r' r' r( forward+ zQwenAudioWarp.forwardNkeyc K s | dddkr tdi }| dd}d|d d| } | j| }
| j| d |
d
} | | jj} | jjdi | d |
i} | jj| d d |
d
}
g }|d |
d}|
| ||fS )N
batch_size !batch decoding is not implementedpromptzL<|startoftranscription|><|en|><|transcribe|><|en|><|notimestamps|><|wo_itn|>zpt)return_tensors
audio_infor5 F)skip_special_tokensr5 r. textr' ) r NotImplementedErrorr
process_audiotor devicegeneratedecoder append)r! data_in data_lengthsr. r frontendr# meta_datar2 queryr5 inputspredresponseresultsresult_ir' r' r( inference0 s$
zQwenAudioWarp.inferenceNNNN) __name__
__module__ __qualname____doc__r r, listrJ
__classcell__r' r' r% r( r
s zQwen/Qwen-Audio-ChatzQwen/QwenAudioChatzQwen-Audio-Chat
QwenAudioChatQwenAudioChatWarpc s< e Zd Z fddZdd Z d defddZ ZS )
rS c s| t ddlm}m} ddlm} |dd}|dd}|dd}|j|d ||d
d } |j|d
d }
| | _ |
| _
d
S )z
Qwen-Audio: Advancing Universal Audio Understanding via Unified Large-Scale Audio-Language Models
https://arxiv.org/abs/2311.07919
Modified from https://github.com/QwenLM/Qwen-Audio
r r r r r bf16Ffp16r T)r rT rU r r Nr ) r! r" r# r r r r$ rT rU r r r% r' r( r X s
zQwenAudioChatWarp.__init__c C r) r* r' r+ r' r' r( r, m r- zQwenAudioChatWarp.forwardNr. c K s | dddkr tdi }| dd}| di } | dd }
|d d ur5| jd |d id
|ig} n|} | jj| j| |
d \} }
|
| d< g }
|d | d }|
| |
|fS )
Nr/ r0 r1 r2 zwhat does the person say?cachehistoryr audior8 )rD rW r7 )r r9 r from_list_formatr chatr? )r! r@ rA r. r rB r# rC r2 rV rW rD rG rH rI r' r' r( rJ r s&
zQwenAudioChatWarp.inferencerK )rL rM rN r r, rP rJ rQ r' r' r% r( rS R s ) dataclassesr typingr r r timenumpynptorchtorch.nn.functionalr
functionalFr whisperfunasr.utils.load_utilsr r funasr.registerr
registerModuler
rS r' r' r' r( s0
<