o iJ@sddlZddlZddlmZddlmZddlmZmZm Z m Z da da Gdddej jZGdddej jZd ed dfd d Zd ed dfd dZGdddejjZGdddeZGdddeZdS)N)Path) find_spec)ListOptionalTupleUnionc @heZdZdZedejdejdejdejdejf ddZed ejde ejejejejffd d Z d S) WKVLinearAttentionEncoder'WKVLinearAttention function definition. time_decay time_firstkeyvaluereturnc Cs|\}}}|tjksJd|dtjd||t|ddks3Jd|d|dt|d|j|_t|  }| }| }| }tj |tj d }t |||||| ||||||S ahWKVLinearAttention function forward pass. Args: time_decay: Channel-wise time decay vector. (D_att) time_first: Channel-wise time first vector. (D_att) key: Key tensor. (B, U, D_att) value: Value tensor. (B, U, D_att) Returns: out: Weighted Key-Value tensor. (B, U, D_att) zCannot process key of length z while context_size is (z). Limit should be increased. rz batch size (z) by dimension (z) should be a multiple of  memory_format)sizewkv_kernel_encoder context_sizemindtype input_dtypetorchexpfloat contiguous empty_likecontiguous_formatforwardsave_for_backward ctxr r r rbatchlengthdimoutr(Y/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/rwkv_bat/rwkv_attention.pyr $    z!WKVLinearAttentionEncoder.forward grad_outputc C|j\}}}}}|j}|\}} } tj|| ftj|j|jd} tj|| ftj|j|jd} tj|tjd} tj|tjd}t |||||| | | | | tj | dd} tj | dd} | | | |fSaWKVLinearAttention function backward pass. Args: grad_output: Output gradient. (B, U, D_att) Returns: grad_time_decay: Gradient for channel-wise time decay vector. (D_att) grad_time_first: Gradient for channel-wise time first vector. (D_att) grad_key: Gradient for key tensor. (B, U, D_att) grad_value: Gradient for value tensor. (B, U, D_att) )rrdevicerr)r&) saved_tensorsrrremptyrrr.rrbackwardrsumr#r+r r r routput grad_dtyper$_r&grad_time_decaygrad_time_firstgrad_key grad_valuer(r(r)r1BH z"WKVLinearAttentionEncoder.backwardN __name__ __module__ __qualname____doc__ staticmethodrTensortensorr rr1r(r(r(r)r ( -r c @r) WKVLinearAttentionDecoderr r r r rrc Cs|\}}}|tjksJd|dtjd||t|ddks3Jd|d|dt|d|j|_t|  }| }| }| }tj |tj d }t ||||||||||||Sr)rwkv_kernel_decoderr wkv_kernelrrrrrrrrrr r!r"r(r(r)r r*z!WKVLinearAttentionDecoder.forwardr+c Cr,r-) r/rrrr0rrr.rrFr1rr2r3r(r(r)r1r;z"WKVLinearAttentionDecoder.backwardNr<r(r(r(r)rErDrErrcddlm}tdurtj|krdStddurtdtjs$tdt t  j dfdd d D}d d d ddd|g}|d||d|da|t_dS)JLoad WKV CUDA kernel. Args: context_size: Context size. rloadNninja}Ninja package was not found. WKV kernel module can't be loaded for training. Please, 'pip install ninja' in your environment.jCUDA is currently a requirement for WKV kernel loading. Please set your devices properly and launch again. cuda_encodercg|]}|qSr(r(.0f kernel_folderr(r) z+load_encoder_wkv_kernel..z wkv_op.cppz wkv_cuda.cu -res-usage--maxrregcount 60--use_fast_math-O3 -Xptxas -O3-DTmax= encoder_wkv_Tnamesourcesverboseextra_cuda_cflags) torch.utils.cpp_extensionrKrrr ImportErrorrcuda is_availabler__file__resolveparentrrK kernel_files kernel_cflagsr(rTr)load_encoder_wkv_kernel6    rocrH)rIrrJNrLrMrN cuda_decodercrPr(r(rQrTr(r)rV8rWz+load_decoder_wkv_kernel..rXrYrZr[r\r]r^ decoder_wkv_Tr`) rerKrFrrrfrrgrhrrirjrkrlr(rTr)load_decoder_wkv_kernelrprscseZdZdZdedededededdf fd d Zdededededdf d d Ze d ej dej dej dej de ej ej ej fde ej e ej ej ej fff ddZ Z S) SelfAttention SelfAttention module definition. Args: size: Input/Output size. attention_size: Attention hidden size. context_size: Context size for WKV kernel. block_id: Block index. num_blocks: Number of blocks in the architecture. rattention_sizeblock_id dropout_rate num_blocksrNcsttjd|_tjt||_tjt||_ tjtdd||_ tjtdd||_ tjtdd||_ tjj ||dd|_tjj ||dd|_tjj ||dd|_tjj ||dd|_||_|||||tjj|d|_dS)!Construct a SelfAttention object.)rrr{T)bias)pN)super__init__rnn ZeroPad2d time_shift Parameterr0r r time_mix_keytime_mix_valuetime_mix_receptanceLinearproj_key proj_valueproj_receptance proj_outputrwreset_parametersDropoutdropout)selfrrvrwrxry __class__r(r)rWs zSelfAttention.__init__c s0||dd||}tdd|}t|D] }|||dd|f<qfddtD}tj||jj|jjd}tjddtD|jj|jjdd} t:||j_ t |jt d | |j_ t |||j_ t ||d |j_ t |d||j_ Wd d S1swYd S) zReset module parameters. Args: size: Block size. attention_size: Attention hidden size. block_id: Block index. num_blocks: Number of blocks in the architecture. r{g?rcs,g|]}dd|dddqS)r{gffffff?g?r()rRhrv ratio_0_to_1r(r)rVsz2SelfAttention.reset_parameters..)rr.cSsg|] }|dddqS)r{r()rRir(r(r)rVsg?g333333?N)ronesrangerCr rr.r no_graddata ones_likemathlogpowrrr) rrrvrwryratio_1_to_almost0 time_weightr decay_speedzigzagr(rr)rus6    "zSelfAttention.reset_parametersr r r rstatecCs|\}}}t| }t|||} t|| } t||| } | || |} | || } t|||}t|||} t||} | | }| || || || |g}||fS)aCompute WKV with state (i.e.: for inference). Args: time_decay: Channel-wise time decay vector. (D_att) time_first: Channel-wise time first vector. (D_att) key: Key tensor. (B, 1, D_att) value: Value tensor. (B, 1, D_att) state: Decoder hidden states. [3 x (B, D_att)] Returns: output: Weighted Key-Value. (B, 1, D_att) state: Decoder hidden states. [3 x (B, 1, D_att)] )rrmaximum)rr r r rr num_state den_state max_statemax_for_outpute1e2 numerator denominator max_for_statewkvr(r(r)wkv_linear_attentions   z"SelfAttention.wkv_linear_attention)r=r>r?r@intrrrrrrBrr __classcell__r(r(rr)rtKsP  /rtcxeZdZdZdededededededd ffd d Z dd ejd e e ejde eje e ejffddZ Z S)DecoderSelfAttentionrurrvrrwrxryrNct|||||dSrzNrrrrrvrrwrxryrr(r)r zDecoderSelfAttention.__init__xrc V|dur |n|ddjf}|j|dj}|j|dj}|j|dj}|}|}t |}|dur||ddjf< j j ||t fdd|ddD\}}|d|ddjf<|d|ddjf<|d|d djf<n tj j ||}|}||}||fS) Compute time mixing. Args: x: SelfAttention input sequences. (B, U, size) state: Decoder hidden states. [5 x (B, 1, D_att, N)] Returns: x: SelfAttention output sequences. (B, U, size) Nr{.c3|] }|djfVqdS.NrwrRsrr(r) z/DecoderSelfAttention.forward..rr)rrwrrrrrrsigmoidrrr r tuplerEapplyrr rrr shifted_xr r receptancer att_stater(rr)r .$   zDecoderSelfAttention.forwardNr=r>r?r@rrrrrBrrrr rr(r(rr)r2  rcr)EncoderSelfAttentionrurrvrrwrxryrNcrrrrrr(r)r&rzEncoderSelfAttention.__init__rrc r) rNr{.c3rrrrrr(r)rTrz/EncoderSelfAttention.forward..rrrr)rrwrrrrrrrrrr r rr rrrrr(rr)r 3rzEncoderSelfAttention.forwardrrr(r(rr)rrr)rrpathlibrimportlib.utilrtypingrrrrrrFautogradFunctionr rErrorsrModulertrrr(r(r(r)s  oo..I