o i4@sdZddlZddlmZddlmZddlmZmZddl Z ddl m Z ddl m Z dd l mZdd lmZdd lmZmZmZmZmZmZd d lmZeeZdaddZGddde jj Z!d,ddZ"d,ddZ#Gddde j$Z%Gddde j$Z&GdddeZ'eGdddeZ(eedd Gd!d"d"eZ)eed#d Gd$d%d%eZ*eGd&d'd'e(Z+ed(d Gd)d*d*e(e Z,gd+Z-dS)-zPyTorch RWKV model.N) dataclass)Path)OptionalUnion)nn)GenerationMixin)GradientCheckpointingLayer)PreTrainedModel) ModelOutputauto_docstringis_bitsandbytes_availableis_ninja_availableis_torch_cuda_availablelogging) RwkvConfigcsddlm}ttjjjddfdddD}tdur'tj|kr'dSt d|d d d d d ddd|g}|d||t t j k|da|t_dS)Nr)loadkernelsrwkvcsg|]}|qSr).0f kernel_folderrZ/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/rwkv/modeling_rwkv.py 4z(load_wkv_cuda_kernel..)z wkv_op.cppz wkv_cuda.cuzwkv_cuda_bf16.cuz2Loading CUDA kernel for RWKV at context length of .z -res-usagez--maxrregcount 60z--use_fast_mathz-O3z -Xptxas -O3z--extra-device-vectorizationz-DTmax=wkv_)namesourcesverboseextra_cuda_cflags) torch.utils.cpp_extensionrr__file__resolveparentrwkv_cuda_kernelmax_seq_lengthloggerinfor get_verbosityDEBUG)context_length load_kernelcuda_kernel_filesflagsrrrload_wkv_cuda_kernel.s*    r2c@s(eZdZedddZedddZdS) RwkvLinearAttentionNFc Cs|\}}} |tjkrtd|dtjd|| t| ddkr4td|d| dt| dd |j|_|jjd ksP|jjd ksP|jjd ksP|jjd krTtd t |  }|jt j krp| }| }| }| }| }| }t j|t jd } |s|dur|durt j|| d t j|jt jd}|dddddfd8<nt jdd|Ddd }|jt jkrtj} ntj} | ||||| |n|jt jkrtjntj} | ||||| |||||| |durddt j|d ddD}| |j|fS)NzCannot process a batch with z+ tokens at the same time, use a maximum of z with this model. rzThe product of batch size (z) and hidden size (z") needs to be a round multiple of rcudazUCalling the CUDA kernel for wkv attention requires all tensors to be on CUDA devices. memory_formatr)dtypedevicer7籡*GcSg|]}|dqSr:) unsqueezersrrrr}z/RwkvLinearAttention.forward..)dimcSr<r=)squeezer?rrrrrA)sizer(r) ValueErrorminr8 input_dtyper9typetorchexpfloat contiguousfloat16 empty_likecontiguous_formatzerosfloat32catbfloat16forward_with_state_bf16forward_with_state forward_bf16forwardsave_for_backwardchunkto) ctx time_decay time_firstkeyvaluestate return_state batch_sizeseq_len hidden_sizeoutput forward_funcrrrrWOsd         zRwkvLinearAttention.forwardc Cs|j}|j\}}}}}tj|tj|tjkrtjntjd} tj|tjd} tj|tjd} tj|tjd} |tjkr>|}|tjkrFt j nt j } | |||||| | | | | | || || || |ddfS)N)r7r8r6)rG saved_tensorsrIrNrOrSrQrMrKr( backward_bf16backwardrLrZ)r[g_outputg_staterGr\r]r^r_re g_time_decay g_time_firstg_keyg_value backward_funcrrrris@ zRwkvLinearAttention.backwardNFN)__name__ __module__ __qualname__ staticmethodrWrirrrrr3Ns  >r3FcCs|\}}}t|}|dur=tj|dddftjd} tj|dddftjd} tj|dddftjdd} n|\} } } t| }t|D]p} |dd| f} |dd| f}t| | |}t| |}t| ||}|| ||}|| |}|||j |dd| f<t| || }t| ||}t| |}|| ||} || |} |} qL|s|dur| | | g}||fS)Nr)r8r;) rDrI zeros_likerQrJrangerKmaximumrZr8)r\r]r^r_r`ra_ seq_lengthre num_state den_state max_state current_index current_key current_valuemax_for_outpute1e2 numerator denominator max_for_staterrrrwkv_linear_attention_cpus4 "       rcCs`tdd||||fD}|ddk}tdus|s|r&t||||||dSt||||||S)Ncss|] }|jjdkVqdS)r5N)r9rH)rtrrr z(rwkv_linear_attention..rr`ra)anyrDr(rr3apply)r\r]r^r_r`rano_cuda one_tokenrrrrwkv_linear_attentions rcs2eZdZd fdd Zd ddZd dd ZZS) RwkvSelfAttentionrcsDt||_tduotj|jk}tr0tr0|s0zt|jWnt y/t dYnw||_ |j }|jdur>|jn|}||_tt||_tt||_ttdd||_ttdd||_ttdd||_td|_tj||dd|_tj||dd|_tj||dd|_tj||dd|_dS)Nz9Could not load the custom CUDA kernel for RWKV attention.rrrrFbias)super__init__configr(r)r.rrr2 Exceptionr*r+layer_idrdattention_hidden_sizer ParameterrIemptyr\r] time_mix_keytime_mix_valuetime_mix_receptance ZeroPad2d time_shiftLinearr^r_ receptancere)selfrr kernel_loadedrdr __class__rrrs0   zRwkvSelfAttention.__init__NcCs|ddkr|dur|ddddd|jf}n||}|dur7|ddddd|jf|dddf<||j|d|j}||j|d|j}||j|d|j}||}||}t | |}|dur|dddf|ddddd|jf<||||fSNrrr) rDrrrrrr^r_rIsigmoidr)rhiddenr`shiftedr^r_rrrrextract_key_values (  ( z#RwkvSelfAttention.extract_key_valueFc sj||d\}}}}|durtfdd|ddDnd}tjj||||d\}}|durb|d|dddddjf<|d|dddddjf<|d|d ddddjf<|||fS) Nr`c3s(|]}|ddddjfVqdSrrrr?rrrr#s&z,RwkvSelfAttention.forward..r:rrrr)rtuplerr\r]rre) rrr` use_cacherr^r_ layer_staterrrrrW!s*    zRwkvSelfAttention.forwardrrrrq)rsrtrurrrW __classcell__rrrrrs rcs(eZdZdfdd ZdddZZS) RwkvFeedForwardrcst||_||_|j}|jdur|jnd|j}td|_t t dd||_ t t dd||_ tj||dd|_tj||dd|_tj||dd|_dS)NrrrFr)rrrrrdintermediate_sizerrrrrIrrrrr^rr_)rrrrdrrrrr6s  zRwkvFeedForward.__init__NcCs |ddkr|dur|ddddd|jf}n||}|dur7|ddddd|jf|dddf<||j|d|j}||j|d|j}tt||}| |}t | |}|dur|dddf|ddddd|jf<|||fSr) rDrrrrrIsquarerelur^r_rr)rrr`rr^rr_rrrrWGs ( ( zRwkvFeedForward.forwardrrrrsrtrurrWrrrrrr5srcs&eZdZfddZdddZZS) RwkvBlockcsvt||_||_|dkrtj|j|jd|_tj|j|jd|_ tj|j|jd|_ t |||_ t |||_dS)Nr)eps)rrrrr LayerNormrdlayer_norm_epsilonpre_lnln1ln2r attentionr feed_forward)rrrrrrr\s  zRwkvBlock.__init__NFcCs||jdkr ||}|j||||d\}}||}|j|||d\}}||}||f}|r8||f7}|S|d7}|S)Nr)r`rrrr)rrrrrr)rrr`routput_attentionsrroutputsrrrrWjs   zRwkvBlock.forward)NFFrrrrrr[s rc@s@eZdZUeed<dZdgZddgZdZdZ de j fdd Z d S) RwkvPreTrainedModelrrrr\r]Tmodulec st|tr|j}|jj}|jj|j||dd||}tjfddt D|j j |j j d}|ddddf}fddt D}tj||j j |j j d}tjddt D|jj |jj dd }||j _t|jtd ||j_t|||j _t||d |j_t|d ||j_dSt|tr|j}|jj}|jjd||}tjfd dt D|j j |j j d}|ddddf}t|||j _t|||j_dSt|tjr5|jjj}d} d} |jdur|jj|d |dkrt|d |d} |d |jjkr&|d|jjkr&d } | | 9} tjj |j| d dSt|tj!rZ|jjj}dtt"|d |d} tjj |j| d dSt|tj#rp|jj$d|jjdSdS)zInitialize the weights.rg?cg|]}|qSrrrirdrrrrz5RwkvPreTrainedModel._init_weights..r8r9Ncs,g|]}dd|dddqS)rgffffff?g?r)rh)r ratio_0_to_1rrrscSsg|] }|dddqS)rrrrrrrrsg?g333333?crrrrrrrrrr)gaing-C6?)% isinstancerrrnum_hidden_layersrdrrItensorrxrr8r9r\r]data ones_likemathlogpowrrrrrweightshaperzero_sqrt vocab_sizeinit orthogonal_ Embeddingmaxrfill_) rrrrratio_1_to_almost0 time_weight decay_speedzigzagrrscaler)rrdrr _init_weightss~          $ z!RwkvPreTrainedModel._init_weightsN) rsrtrur__annotations__base_model_prefix_no_split_modules_keep_in_fp32_modulessupports_gradient_checkpointing _is_statefulrModulerrrrrr}s rz+ Class for the RWKV model outputs. ) custom_introc@sneZdZUdZdZeejed<dZ ee ejed<dZ ee ejdfed<dZ ee ejdfed<dS) RwkvOutputa  state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. Nlast_hidden_stater`. hidden_states attentions)rsrtru__doc__rrrI FloatTensorrr`listrrrrrrrrs rzK Base class for causal language model (or autoregressive) outputs. c@seZdZUdZdZeejed<dZ eejed<dZ ee ejed<dZ ee ejdfed<dZee ejdfed<dS) RwkvCausalLMOutputap loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. Nlosslogitsr`.rr)rsrtrurrrrIrrrr`rrrrrrrrrs  rcseZdZfddZddZddZe        ddeej d eej d eej d ee ej d ee d ee dee dee de eeffddZddZddZZS) RwkvModelcsdttjj|_tfddtj D|_ t j|_ d|_ d|_|dS)Ncsg|]}t|dqS)r)r)ridxrrrrsz&RwkvModel.__init__..F)rrrrrrd embeddings ModuleListrxrblocksrln_outlayers_are_rescaledgradient_checkpointing post_initrrrrrrs   zRwkvModel.__init__cC|jSrrrrrrrget_input_embeddings zRwkvModel.get_input_embeddingscC ||_dSrrr rnew_embeddingsrrrset_input_embeddings zRwkvModel.set_input_embeddingsN input_idsattention_mask inputs_embedsr`rroutput_hidden_states return_dictreturnc s|dur|n|jj}|dur|n|jj}|dur|n|js!|jjnd}|dur)|n|jj}|dur6td|j|jkr@| |durLdurLt d|durXdurXt ddura| ||r|dur d|jj |jjffddtd D}|d d 8<|jr|jr|rtd d}} |rd nd} |rd nd} t|jD]4\} } | | |||d\} }}|jr|jjdkr| d|jjdkr| d} |r| | f} |r| |f} q|| } |r| | f} |stdd| || | fDSt| || | dS)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. NFz<`attention_mask` was passed, but it is unused in this model.zDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrcs0g|]}tj|dkrjntjjdqS)rr)rIrPr8rQr9rrrrrrIs z%RwkvModel.forward..rgꌠ9Y>)FzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...r)r`rrrr:css|] }|dur|VqdSrrr)rxrrrrtrz$RwkvModel.forward..)rr`rr)rrrtrainingruse_return_dictr* warning_oncer_rescale_layersrErrDrdrrxr enumerater rescale_everyrrr)rrrrr`rrrrrall_self_attentionsall_hidden_statesrblockrrrrrWsn              zRwkvModel.forwardcCsx|j|j kr dS|jjdkrtt|jD]\}}|jrA|jj j dt ||jj|j jj dt ||jjqt|jj j drl|jj j jdt ||jj|j jj jdt ||jjqt|jj j dr||jj |||j j|q|jj j dt ||jj|j jj dt ||jjqWdn1swY|j |_dS)Nrr:SCB quant_state)rrrr rIno_gradrrrrermul_intrr_hasattrr$div_ _bnb_4bit_dequantize_and_rescale)rblock_idr#rrrr}s&   ""$ "zRwkvModel._rescale_layerscCsttstdddl}|j|jj|jj}|dt ||j j |j j |ddd|j}t|d|dS) z Perform the dequantization and rescaling of the weights of a given layer. After that operation the layer will be quantized again. z/Please install bitsandbytes to use this method.rNr:cpuF) requires_gradr)r ImportError bitsandbytes functionaldequantize_4bitrrr%r*r(rr r Params4bitrZr9setattr)r target_layerr,bnbdequant_weights quant_weightrrrr+sz*RwkvModel._bnb_4bit_dequantize_and_rescale)NNNNNNNN)rsrtrurr rr rrI LongTensorrrboolrrrrWrr+rrrrrrsD    irz The RWKV Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). cseZdZdgZfddZddZddZdd d Ze         dd e e j d e e j d e e j de e e j de e j de ede ede ede edeeeffddZZS)RwkvForCausalLMz head.weightcs8t|t||_tj|j|jdd|_| dS)NFr) rrrrrrrdrheadrrrrrrs   zRwkvForCausalLM.__init__cCr rrr<rrrrget_output_embeddingsr z%RwkvForCausalLM.get_output_embeddingscCr rrr=rrrrset_output_embeddingsrz%RwkvForCausalLM.set_output_embeddingsNc Ksv|dur|dddfd}|dur|durd|i}nd|i}||d<||d<|D] \}}||vr8|||<q,|S)Nrrrr`r)r>items) rrr`rrkwargs model_inputsr^r_rrrprepare_inputs_for_generations z-RwkvForCausalLM.prepare_inputs_for_generationrrrr`labelsrrrrrc  Ks| dur| n|jj} |j||||||| d} | d} || } d}|dur3|j| |fd|jji| }| sI| f| dd}|durG|f|S|St|| | j| j| j dS)aJ input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. N)rr`rrrrrrr)rrr`rr) rrrr< loss_functionrrr`rr)rrrrr`rDrrrrrA rwkv_outputsrrrrerrrrWs@% zRwkvForCausalLM.forward)NNN) NNNNNNNNN)rsrtru_tied_weights_keysrr>r?rCr rrIr9rrr:rrrrWrrrrrr;sJ      r;)r;rrrq).rr dataclassesrpathlibrtypingrrrIr generationrmodeling_layersr modeling_utilsr utilsr r r rrrconfiguration_rwkvr get_loggerrsr*r(r2autogradFunctionr3rrrrrrrrrrr;__all__rrrrsR          j , F&"T . r