o Xi%@s.ddlmZddlmZddlmZmZddlZddlZddl m m Z ddlm Z ddlm Z ddlmZdd lmZmZeGd d d ZGd d d e jZGddde jZGddde jZdddZGddde jZGddde jZGddde jZGddde jZGddde jZdS) ) dataclass)Dict)IterableOptionalN)Tensor)nn) transcribe)detect_languagedecodec@s^eZdZUeed<eed<eed<eed<eed<eed<eed<eed<eed <eed <d S) ModelDimensionsn_mels n_audio_ctx n_audio_state n_audio_head n_audio_layern_vocab n_text_ctx n_text_state n_text_head n_text_layerN)__name__ __module__ __qualname__int__annotations__rrA/home/ubuntu/.local/lib/python3.10/site-packages/whisper/model.pyr s  r cs&eZdZdedeffdd ZZS) LayerNormxreturncst||jSN)superforwardfloattypedtypeselfr __class__rrr#szLayerNorm.forward)rrrrr# __classcell__rrr)rrsrc@seZdZdedefddZdS)Linearrr cCs2t||j|j|jdurdS|j|jSr!)Flinearweighttor&biasr'rrrr##s  zLinear.forwardN)rrrrr#rrrrr,"sr,cs2eZdZdededeedeffdd ZZS)Conv1drr/r1r cs.t|||j|durdS||jSr!)r" _conv_forwardr0r&)r(rr/r1r)rrr3*s  zConv1d._conv_forward)rrrrrr3r+rrr)rr2)s*r2'cCs|ddksJt||dd}t| t|d}t|ddtjf|tjddf}tjt|t|gddS)z*Returns sinusoids for positional embeddingrrNdim) nplogtorchexparangenewaxiscatsincos)lengthchannels max_timescalelog_timescale_incrementinv_timescales scaled_timerrr sinusoids0s *rGc steZdZdedeffdd Z   ddedeedeed eefd d Zdd ed ededeefddZ Z S)MultiHeadAttentionn_staten_headcsHt||_t|||_t||dd|_t|||_t|||_dS)NF)r1)r"__init__rJr,querykeyvalueout)r(rIrJr)rrrK:s   zMultiHeadAttention.__init__Nrxamaskkv_cachec Cs||}|dus|dus|j|vr)||dur|n|}||dur%|n|}n ||j}||j}|||||}||Sr!)rLrMrN qkv_attentionrO) r(rrPrQrRqkvwvrrrr#Bs    zMultiHeadAttention.forwardrTrUrVc Cs|j\}}}||jd}|jg|jdd|jdRdddd|}|jg|jdd|jdRdddd|}|jg|jdd|jdRdddd}||} |durm| |d|d|f} tj| dd|j} | |ddddj ddS) Ngпr5rrr6) start_dim) shaperJviewpermuter-softmaxr$r0r&flatten) r(rTrUrVrQn_batchn_ctxrIscaleqkwrrrrSXs 440z MultiHeadAttention.qkv_attentionNNNr!) rrrrrKrrdictr#rSr+rrr)rrH9s  (rHc sZeZdZddededeffdd Z   dded eed eed eefd d Z Z S)ResidualAttentionBlockFrIrJcross_attentioncs|tt|||_t||_|rt||nd|_|r t|nd|_|d}t t ||t t |||_ t||_ dS)N)r"rKrHattnrattn_ln cross_attn cross_attn_lnr Sequentialr,GELUmlpmlp_ln)r(rIrJrhn_mlpr)rrrKhs    zResidualAttentionBlock.__init__NrrPrQrRcCsR||j||||d}|jr||j||||d}||||}|S)NrQrR)rR)rjrkrlrmrprq)r(rrPrQrRrrrr#us zResidualAttentionBlock.forward)Fre) rrrrboolrKrrrfr#r+rrr)rrggsrgc s@eZdZdededededef fdd Zdefd d ZZS) AudioEncoderr rarIrJn_layercsptt|ddd|_tdddd|_|dt|tfddt |D|_ t |_ dS) NrYr) kernel_sizepaddingr5)rwstriderxpositional_embeddingcsg|]}tqSrrg.0_rJrIrr sz)AudioEncoder.__init__..) r"rKr2conv1conv2register_bufferrGr ModuleListrangeblocksrln_post)r(r rarIrJrvr)rrrKs zAudioEncoder.__init__rcCst||}t||}|ddd}|jdd|jjks&Jd||j|j}|j D]}||}q2| |}|S)zt x : torch.Tensor, shape = (batch_size, n_mels, n_ctx) the mel spectrogram of the audio rr5rNzincorrect audio shape) r-gelurrr]r[rzr0r&rr)r(rblockrrrr#s   zAudioEncoder.forward)rrrrrKrr#r+rrr)rrus" ruc sNeZdZdededededef fdd Zdd ed ed eefd d ZZ S) TextDecoderrrarIrJrvcstt||_tt||_t fddt |D|_ t |_ t||tj d}|jd|dddS)Ncsg|] }tddqS)T)rhr{r|rrrrsz(TextDecoder.__init__..rrQF) persistent)r"rKr Embeddingtoken_embedding Parameterr:emptyrzrrrrlnfill_r8inftriu_r)r(rrarIrJrvrQr)rrrKs  zTextDecoder.__init__NrrPrRcCs|r tt|jdnd}|||j|||jd}||j}|jD] }||||j |d}q)| |}|t |jj |jdd}|S)z x : torch.LongTensor, shape = (batch_size, <= n_ctx) the text tokens xa : torch.Tensor, shape = (batch_size, n_mels, n_audio_ctx) the encoded audio features to be attended on rrrXrs)nextitervaluesr[rrzr0r&rrQrr: transposer/r$)r(rrPrRoffsetrlogitsrrrr#s"   "zTextDecoder.forwardr!) rrrrrKrrrfr#r+rrr)rrs"$rcseZdZdeffdd ZdejfddZdejdejfd d Zdejdejd e e ejffd d Z e ddZ e ddZddeefddZeZeZeZZS)Whisperdimscs`t||_t|jj|jj|jj|jj|jj|_ t |jj |jj |jj |jj|jj|_dSr!)r"rKrrur rrrrencoderrrrrrrdecoder)r(rr)rrrKs  zWhisper.__init__melcCs ||Sr!)r)r(rrrr embed_audios zWhisper.embed_audiotokensaudio_featurescCs |||Sr!)r)r(rrrrrrs zWhisper.logitsr cCs||||Sr!)rr)r(rrrrrr#szWhisper.forwardcCst|jSr!)r parametersdevicer(rrrrszWhisper.devicecCs |jjdkS)Ni)rrrrrris_multilinguals zWhisper.is_multilingualNcachecsPdurinigfdddtjffdd }j|fS)a The `MultiHeadAttention` module optionally accepts `kv_cache` which stores the key and value tensors calculated for the previous positions. This method returns a dictionary that stores all caches, and the necessary hooks for the key and value projection modules that save the intermediate tensors to be reused during later calculations. Returns ------- cache : Dict[nn.Module, torch.Tensor] A dictionary object mapping the key/value projection modules to its cache hooks : List[RemovableHandle] List of PyTorch RemovableHandle objects to stop the hooks to be called NcsV|vs|jdjjjdkr||<|Stj||gdd|<|S)Nrrr6)r[rrzr:r>detach)moduler~output)rr(rr save_to_caches z5Whisper.install_kv_cache_hooks..save_to_cachelayercs6t|tr|j|jdSdSr!) isinstancerHappendrMregister_forward_hookrN)r)hooksrrr install_hookss z5Whisper.install_kv_cache_hooks..install_hooks)rModulerapply)r(rrr)rrrr(rinstall_kv_cache_hookss  zWhisper.install_kv_cache_hooksr!)rrrr rKr:rrrrstrr#propertyrrrrfrdetect_language_functionr transcribe_functionr decode_functionr r+rrr)rrs$    r)r4) dataclassesrtypingrrrnumpyr8r:torch.nn.functionalr functionalr-rr rdecodingr rr rr rr,r2rGrrHrgrurrrrrrs*       .#