o
    ei                     @   s  d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ e$,e-Z.edG dd dej/Z0G dd dZ1dd Z2eddKddZ3dej4de5d ej4fd!d"Z6	#dLd$ej/d%ej4d&ej4d'ej4d(ej4dB d)e7d*e7d+ee! fd,d-Z8ee3G d.d/ d/ej/Z9G d0d1 d1ej/Z:G d2d3 d3ej/Z;eG d4d5 d5ej/Z<G d6d7 d7ej/Z=G d8d9 d9eZ>G d:d; d;eZ?G d<d= d=eZ@e>e?d>ZAe"G d?d@ d@e@ZB		A	dMdBej4eCej4 B dB dCe5dB d(ej4dB d ej4e5B fdDdEZDe"G dFdG dGe@eZEG dHdI dIee@ZFg dJZGdS )N    )Callable)AnyN)nn   )initialization)ACT2FN)GenerationMixin)lazy_load_kerneluse_experts_implementationuse_kernel_forward_from_hubuse_kernel_func_from_hubuse_kernelized_func)create_causal_mask) GenericForSequenceClassificationGradientCheckpointingLayer)MoeCausalLMOutputWithPastMoeModelOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)OutputRecordercapture_outputs   )JambaConfigRMSNormc                       sF   e Zd Zddeddf fddZdejdejfdd	Zd
d Z  Z	S )JambaRMSNormư>epsreturnNc                    s&   t    tt|| _|| _dS )z;
        JambaRMSNorm is equivalent to T5LayerNorm
        N)super__init__r   	Parametertorchonesweightvariance_epsilon)selfhidden_sizer"   	__class__ f/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/jamba/modeling_jamba.pyr%   9   s   

zJambaRMSNorm.__init__hidden_statesc                 C   sJ   |j }|tj}|djddd}|t|| j  }| j|| S )N   T)keepdim)	dtypetor'   float32powmeanrsqrtr*   r)   )r+   r1   input_dtypevariancer/   r/   r0   forwardA   s
   zJambaRMSNorm.forwardc                 C   s   t | jj d| j S )Nz, eps=)tupler)   shaper*   r+   r/   r/   r0   
extra_reprH   s   zJambaRMSNorm.extra_repr)r!   )
__name__
__module____qualname__floatr%   r'   Tensorr=   rA   __classcell__r/   r/   r-   r0   r    7   s    r    c                   @   s   e Zd ZdZdZejdfddZdd Zdd	 Z		dd
ej
dej
dedeeef dB deej
ej
f f
ddZdejfddZdej
dedeeef fddZddedB defddZdS ) HybridMambaAttentionDynamicCachea  
    A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache
    (which has a constant shape regardless of seq_len).

    This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states`
    and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor
    For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`,
    while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors).
    For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors),
    while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`,
    and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`.
    FNc           	   
      s&  || _ |j| _d| _|j|j }|j}|j}g | _g | _g | _	t
|jD ]P}| j| dkrN|  jtj |||dg7  _|  jtj |||dg7  _q$|  jtjg g  dg7  _|  jtjg g  dg7  _| j	| q$ fddt
|jD | _ fddt
|jD | _d S )NFmambadevicer5   rK   c                        g | ]}t jg g  d qS rL   r'   tensor.0_
batch_sizerK   r/   r0   
<listcomp>s        z=HybridMambaAttentionDynamicCache.__init__.<locals>.<listcomp>c                    rM   rN   rO   rQ   rT   r/   r0   rV   t   rW   )r5   layers_block_typehas_previous_statemamba_expandr,   mamba_d_statemamba_d_convconv_states
ssm_statestransformer_layersrangenum_hidden_layersr'   zerosrP   append	key_cachevalue_cache)	r+   configrU   r5   rK   intermediate_sizessm_state_sizeconv_kernel_sizeir/   rT   r0   r%   \   s,   
   z)HybridMambaAttentionDynamicCache.__init__c                 C   s
   t | jS N)lenrd   r@   r/   r/   r0   __len__v   s   
z(HybridMambaAttentionDynamicCache.__len__c                 C   s   | j | | j| fS rk   )rd   re   r+   	layer_idxr/   r/   r0   __getitem__y   s   z,HybridMambaAttentionDynamicCache.__getitem__
key_statesvalue_statesro   cache_kwargsr#   c                 C   sz   | j | jd dkr|| j |< || j|< ntj| j | |gdd| j |< tj| j| |gdd| j|< | j | | j| fS )Nr3   r   r2   dim)rd   r?   re   r'   cat)r+   rq   rr   ro   rs   r/   r/   r0   update|   s   
z'HybridMambaAttentionDynamicCache.updatebeam_idxc                 C   s   |   dkrdtt| jD ]X}| j| j}| j| d||| j|< | j| j}| j| d||| j|< | j| j}| j| d||| j|< | j	| j}| j	| d||| j	|< qdS dS )zDReorders the cache for beam search, given the selected beam indices.r   N)
get_seq_lengthr`   rl   rd   rK   index_selectr6   re   r]   r^   )r+   rx   ro   rK   r/   r/   r0   reorder_cache   s    z.HybridMambaAttentionDynamicCache.reorder_cachecache_positionc                 C   s$   d}|j d }| || }||fS )zDReturn the length and offset of the cache, used to generate the maskr   )r?   ry   )r+   r|   ro   	kv_offsetquery_length	kv_lengthr/   r/   r0   get_mask_sizes   s   
z/HybridMambaAttentionDynamicCache.get_mask_sizesr   c                 C   sN   || j vr
| j d n|}t| j|ks| j| jd dkrdS | j| jd S )zYReturns the sequence length of the cached states. A layer index can be optionally passed.r   r3   )r_   rl   rd   r?   rn   r/   r/   r0   ry      s   "z/HybridMambaAttentionDynamicCache.get_seq_lengthrk   )r   )rB   rC   rD   __doc__is_compileabler'   float16r%   rm   rp   rF   intdictstrr   r>   rw   
LongTensorr{   r   ry   r/   r/   r/   r0   rH   L   s*    
 rH   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..Nr3   r2   rt   )r?   r'   rv   )xx1x2r/   r/   r0   rotate_half   s   r   rotary_pos_embc                 C   sD   | |}| |}| | t| |  }|| t||  }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkcossinunsqueeze_dimq_embedk_embedr/   r/   r0   apply_rotary_pos_emb   s
   

r   r1   n_repr#   c                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r?   expandreshape)r1   r   batchnum_key_value_headsslenhead_dimr/   r/   r0   	repeat_kv   s
   0r           modulequerykeyvalueattention_maskscalingdropoutkwargsc                 K   s   t || j}t || j}	t||dd| }
|d ur |
| }
tjj|
dtjd	|j
}
tjj|
|| jd}
t|
|	}|dd }||
fS )Nr2   r   r3   ru   r5   )ptrainingr   )r   num_key_value_groupsr'   matmul	transposer   
functionalsoftmaxr7   r6   r5   r   r   
contiguous)r   r   r   r   r   r   r   r   rq   rr   attn_weightsattn_outputr/   r/   r0   eager_attention_forward   s   
r   c                       sz   e Zd ZdZdedef fddZ			ddejdejdB d	e	dB d
ej
dB dee deejejdB f fddZ  ZS )JambaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrf   ro   c                    s   t    || _|| _t|d|j|j | _|j|j | _	| jd | _
|j| _d| _tj|j|j| j dd| _tj|j|j| j dd| _tj|j|j| j dd| _tj|j| j |jdd| _d S )Nr   g      TFbias)r$   r%   rf   ro   getattrr,   num_attention_headsr   r   r   r   attention_dropout	is_causalr   Linearq_projk_projv_projo_proj)r+   rf   ro   r-   r/   r0   r%      s   
 zJambaAttention.__init__Nr1   r   past_key_valuesr|   r   r#   c                 K   s   |j d d }g |d| jR }| ||dd}| ||dd}	| ||dd}
|d urF||	|
| jd|i\}	}
t	
| jjt}|| ||	|
|f| jsZdn| j| jd|\}}|jg |dR   }| |}||fS )Nr3   r   r2   r|   r   )r   r   )r?   r   r   viewr   r   r   rw   ro   r   get_interfacerf   _attn_implementationr   r   r   r   r   r   r   )r+   r1   r   r   r|   r   input_shapehidden_shapequery_statesrq   rr   attention_interfacer   r   r/   r/   r0   r=     s6   

zJambaAttention.forwardNNN)rB   rC   rD   r   r   r   r%   r'   rF   rH   r   r   r   r>   r=   rG   r/   r/   r-   r0   r      s&    r   c                       s   e Zd ZdZdef fddZ		ddejdedB dej	dB fd	d
Z
ddedB dej	dB fddZ		ddedB dej	dB fddZ  ZS )JambaMambaMixeru  
    Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    rf   c                    s  t    || _|| _|j| _|j| _|j| _|j	|j | _
|j| _|j| _|j| _tj| j
| j
| j| j| j
| jd d| _|j| _t|j | _tj| j| j
d | jd| _tj| j
| j| jd  dd| _tj| j| j
dd| _td| jd d d d f }|| j
d }t t!|| _"t t#| j
| _$tj| j
| j| jd| _%t&| j|j'd| _(t&| j|j'd| _)t&| j|j'd| _*t+d	}t,|d
d a-t,|dd a.t+d}t,|dd a/t,|dd a0t,|dd a1t2t/t1t.t-t0fa3t3st45d d S d S )Nr   )in_channelsout_channelsr   kernel_sizegroupspaddingr2   r   FTr3   r"   zcausal-conv1dcausal_conv1d_updatecausal_conv1d_fnz	mamba-ssmselective_state_updatemamba_inner_fnselective_scan_fna  The fast path is not available because on of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d.)6r$   r%   rf   ro   r,   r[   rh   r\   ri   rZ   rg   mamba_dt_ranktime_step_rankmamba_conv_biasuse_conv_biasmamba_proj_biasuse_biasr   Conv1dconv1d
hidden_act
activationr   actr   in_projx_projdt_projr'   aranger   r   r&   logA_logr(   Dout_projr    rms_norm_epsdt_layernormb_layernormc_layernormr	   r   r   r   r   r   r   allis_fast_path_availableloggerwarning_once)r+   rf   ro   Acausal_conv1d	mamba_ssmr-   r/   r0   r%   3  s\   
	 zJambaMambaMixer.__init__Nr1   cache_paramsr   c                 C   s  |j \}}}|d uo*|jo*|dko*|j| j j d |j| j j d   ko(|kn  }| |dd}|jddd\}}	|d urH||d }| j	j
| j	j
d| j	j
d}
|rtt|d|j| j |
| j	j| j}|d}n'|d urtj|| j|j d  df}|j| j | t||
| j	j| jd}|d ur||d }| |dd}tj|| j| j| jgdd\}}}| |}| |}| |}| jjj }t!  t"| jjj | jj_ W d    n1 sw   Y  | |dd}t!  || jj_ W d    n	1 sw   Y  t#| j$%  }|d ur(|% nd }|rVt&|j| j |d |d ||d d df |d d df | j'|	d |dd	
d}n.t(||||dd|dd| j'% |	|ddd

\}}|d ur|d ur|j| j | | )|dd}|S )Nr   r   r2   rt   r3   )r   ).r   T)dt_softplus)delta_softplusreturn_last_state)*r?   rY   r]   ro   r^   r   r   chunkr   r   r)   r   sizer   squeezer   r   r   r   padri   copy_r   r   r'   splitr   rh   r   r   r   r   datano_grad
zeros_likeexpr   rE   r   r   r   r   )r+   r1   r   r   rU   seq_lenrS   use_precomputed_statesprojected_statesgateconv_weightsr]   ssm_parameters	time_stepBCtime_proj_biasdiscrete_time_stepr   scan_outputs	ssm_statecontextualized_statesr/   r/   r0   cuda_kernels_forwardt  s   	$










z$JambaMambaMixer.cuda_kernels_forwardc              	   C   s  |j \}}}|j}| |dd}|jddd\}	}
|d ur&|	|d }	t|t}|r|j| j	 j d |kr| j
rD|j| j	  }n|j| j	 }||	j}|jr|dkr|j| j	 j d |kr|j| j	 }tj|ddd}|	d d d d df |d d d d df< ||j| j	< tj|| jjd d dd d f  dd}	| jr|	| jj7 }	| |	|d}	n@tj|	| j|	j d  df}||j| j	< | | |	dd |f }	ntj|| j| jf|	j|d}| | |	dd |f }	|d ur|	|d }	| |	dd}tj || j!| j| jgdd\}}}| "|}| #|}| $|}| %|}tj&|dd}t'| j()  }t'|d d d d d d f |d d d d d d d f  }|d d d d d d d f |d d d d d d d f )  }||	d d d d d d d f )  }g }t*|D ]D}|d d d d |d d f | |d d d d |d d f  }t+|||d d |d d f d}|,|d d d d df  qtj-|dd}||	| j.d d d d f   }|| |
 }|r||j| j	< | /|dd}|S )	Nr   r2   rt   r   r3   )shiftsdims.rJ   )0r?   r5   r   r   r   r   
isinstancerH   r^   ro   r   cloner6   rK   rY   r]   r'   rollsumr   r)   r   r   r   r   r   r   ri   rb   rg   rh   r   r   r   r   r   r   r   softplusr   r   rE   r`   r   rc   stackr   r   )r+   input_statesr   r   rU   r  rS   r5   r  r1   r  	use_cacher  
conv_stater  r  r  r	  r  r   
discrete_A
discrete_BdeltaB_ur  rj   scan_outputr  r/   r/   r0   slow_forward  sx   
((



:<$<* zJambaMambaMixer.slow_forwardc                 C   sR   | j jrtrd| jjjjvrtd d| j _| j jr"| 	|||S | 
|||S )NcudazFast Mamba kernels are not available. Make sure that they are installed and that the mamba module is on a CUDA device. Turning off the fast path `config.use_mamba_kernels=False` and falling back to the slow path.F)rf   use_mamba_kernelsr   r   r)   rK   typer   r   r  r  )r+   r1   r   r   r/   r/   r0   r=   4  s   zJambaMambaMixer.forward)NN)rB   rC   rD   r   r   r%   r'   rF   rH   r   r  r  r=   rG   r/   r/   r-   r0   r   +  s(    D
kXr   c                       s$   e Zd Z fddZdd Z  ZS )JambaMLPc                    sr   t    || _|j| _|j| _tj| j| jdd| _tj| j| jdd| _tj| j| jdd| _	t
|j | _d S NFr   )r$   r%   rf   r,   rg   r   r   	gate_projup_proj	down_projr   r   act_fnr+   rf   r-   r/   r0   r%   J  s   
zJambaMLP.__init__c                 C   s$   |  | | || | }|S rk   )r'  r(  r%  r&  )r+   r   r'  r/   r/   r0   r=   T  s    zJambaMLP.forward)rB   rC   rD   r%   r=   rG   r/   r/   r-   r0   r#  I  s    
r#  c                       sH   e Zd ZdZdef fddZdejdejdejdejfd	d
Z  Z	S )JambaExpertsz2Collection of expert weights stored as 3D tensors.rf   c                    sn   t    |j| _|j| _|j| _t	t
| jd| j | j| _t	t
| j| j| j| _t|j | _d S )Nr2   )r$   r%   num_local_expertsnum_expertsr,   
hidden_dimrg   intermediate_dimr   r&   r'   emptygate_up_projr'  r   r   r(  r)  r-   r/   r0   r%   ]  s   
 zJambaExperts.__init__r1   top_k_indextop_k_weightsr#   c                 C   s  t |}t  % t jjj|| jd}|ddd}t |j	ddd
 }W d    n1 s1w   Y  |D ]O}|d }|| jkrDq8t || \}}	||	 }
tj|
| j| jddd\}}| || }tj|| j| }|||	|d f  }|d|	||j q8|S )N)num_classesr2   r   r   )r3   r   rt   r3   )r'   r   r   r   r   one_hotr,  permutegreaterr  nonzerowherelinearr0  r   r(  r'  
index_add_r6   r5   )r+   r1   r1  r2  final_hidden_statesexpert_mask
expert_hit
expert_idx	top_k_pos	token_idxcurrent_stater  upcurrent_hidden_statesr/   r/   r0   r=   f  s$   


"zJambaExperts.forward)
rB   rC   rD   r   r   r%   r'   rF   r=   rG   r/   r/   r-   r0   r*  Y  s    	r*  c                       sD   e Zd ZdZdef fddZdd Zdejdejfd	d
Z	  Z
S )JambaSparseMoeBlocka  
    This implementation is
    strictly equivalent to standard MoE with full capacity (no
    dropped tokens). It's faster since it formulates MoE operations
    in terms of block-sparse operations to accommodate imbalanced
    assignments of tokens to experts, whereas standard MoE either
    (1) drop tokens at the cost of reduced performance or (2) set
    capacity factor to number of experts and thus waste computation
    and memory on padding.
    rf   c                    sN   t    |j| _|j| _|j| _|j| _t	j
| j| jdd| _t|| _d S r$  )r$   r%   r,   r-  rg   ffn_dimr,  num_experts_per_toktop_kr   r   routerr*  expertsr)  r-   r/   r0   r%     s   
zJambaSparseMoeBlock.__init__c                 C   s<   t jjj|dt jd}t j|| jdd\}}|||jfS )Nr3   r   rt   )	r'   r   r   r   rE   topkrG  r6   r5   )r+   r1   router_logitsrouting_weightsr2  r1  r/   r/   r0   route_tokens_to_experts  s   z+JambaSparseMoeBlock.route_tokens_to_expertsr1   r#   c                 C   sR   |j \}}}|d|}| |}| ||\}}| |||}||||}|S )Nr3   )r?   r   rH  rM  rI  r   )r+   r1   rU   sequence_lengthr-  rK  r1  r2  r/   r/   r0   r=     s   
zJambaSparseMoeBlock.forward)rB   rC   rD   r   r   r%   rM  r'   rF   r=   rG   r/   r/   r-   r0   rD    s
    
rD  c                       s~   e Zd Zdedef fddZ					ddejdejdB d	ejdB d
e	dB de
dB dejdB dee dejfddZ  ZS )JambaAttentionDecoderLayerrf   ro   c                    sl   t    |jr|j| nd}t||| _|dkrtnt}||| _t|j	|j
d| _t|j	|j
d| _d S )Nr   r   )r$   r%   layers_num_expertsr   	self_attnrD  r#  feed_forwardr    r,   r   input_layernormpre_ff_layernormr+   rf   ro   r,  ffn_layer_classr-   r/   r0   r%     s   

z#JambaAttentionDecoderLayer.__init__NFr1   r   position_idsr   r  r|   r   r#   c           
   	   K   s\   |}|  |}| jd||||||d|\}}	|| }|}| |}| |}|| }|S )N)r1   r   rW  r   r  r|   r/   )rS  rQ  rT  rR  )
r+   r1   r   rW  r   r  r|   r   residualrS   r/   r/   r0   r=     s$   


	

z"JambaAttentionDecoderLayer.forward)NNNFN)rB   rC   rD   r   r   r%   r'   rF   r   rH   boolr   r   FloatTensorr=   rG   r/   r/   r-   r0   rO    s0    	rO  c                       sh   e Zd Zdedef fddZ			ddejdejdB dejdB d	e	dB d
e
e dejfddZ  ZS )JambaMambaDecoderLayerrf   ro   c                    sn   t    |jr|j| nd}t||d| _|dkrtnt}||| _t|j	|j
d| _t|j	|j
d| _d S )Nr   )rf   ro   r   )r$   r%   rP  r   rI   rD  r#  rR  r    r,   r   rS  rT  rU  r-   r/   r0   r%     s   

zJambaMambaDecoderLayer.__init__Nr1   r   rW  r   r   r#   c                 K   sJ   |}|  |}| j|||d}|| }|}| |}| |}|| }|S )N)r1   r   r   )rS  rI   rT  rR  )r+   r1   r   rW  r   r   rX  r/   r/   r0   r=     s   


zJambaMambaDecoderLayer.forwardr   )rB   rC   rD   r   r   r%   r'   rF   r   rH   r   r   rZ  r=   rG   r/   r/   r-   r0   r[    s$    r[  c                       sh   e Zd ZU eed< dZdZddgZdZdZ	dZ
dZeegeeejddd	Ze  fd
dZ  ZS )JambaPreTrainedModelrf   modelTrO  r[  r   rH  )
layer_name)r1   
attentionsrK  c                    s   t  | t|tr5td|jd d d d f }||jd	 }t
|jt| t
|j d S t|trRt
j|jd| jjd t
j|jd| jjd d S d S )Nr   r3   r   )r9   std)r$   _init_weightsr  r   r'   r   rh   r   rg   r   initr   r   r   ones_r   r*  normal_r0  rf   initializer_ranger'  )r+   r   r   r-   r/   r0   ra    s   

z"JambaPreTrainedModel._init_weights)rB   rC   rD   r   __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_is_statefulrO  r[  r   r   r   r   _can_record_outputsr'   r   ra  rG   r/   r/   r-   r0   r\    s   
 r\  )	attentionrI   c                       s   e Zd Zdef fddZeee							ddej	dB dej
dB dej	dB dedB d	ejdB d
edB dej	dB dee defddZdd Z  ZS )
JambaModelrf   c                    s   t  | |j| _|j| _t|j|j| j| _g }t	|j
D ]}t|j|  }||||d q t|| _t|j|jd| _d| _|   d S )N)ro   r   F)r$   r%   pad_token_idpadding_idx
vocab_sizer   	Embeddingr,   embed_tokensr`   ra   ALL_DECODER_LAYER_TYPESrX   rc   
ModuleListlayersr    r   final_layernormgradient_checkpointing	post_init)r+   rf   decoder_layersrj   layer_classr-   r/   r0   r%     s   zJambaModel.__init__N	input_idsr   rW  r   inputs_embedsr  r|   r   r#   c              	   K   s"  |d u |d uA rt d|d u r| |}|r)|d u r)t| j|jd |j|jd}|d u rE|d ur5| nd}	tj	|	|	|jd  |jd}|d u rN|
d}t| j|||||d}
| ||}|}| jD ]}t|trm|n|
}||f|||||d|}qd| |}|r|jsd|_t||d	S )
Nz:You must specify exactly one of input_ids or inputs_embedsr   )rf   rU   r5   rK   r   rL   )rf   r  r   r|   r   rW  )r   rW  r   r  r|   T)last_hidden_stater   )
ValueErrorru  rH   rf   r?   r5   rK   ry   r'   r   r   r   _update_mamba_maskrx  r  r[  ry  rY   r   )r+   r~  r   rW  r   r  r  r|   r   past_seen_tokenscausal_mask
mamba_maskr1   decoder_layer
layer_maskr/   r/   r0   r=   "  s`   





zJambaModel.forwardc                 C   s6   |}|dur|d dks|durt |dkrd}|S )zv
        No need for zeroing states when
            1. Cached forward
            2. Attending to all inputs
        Nr   r   )r'   r   )r+   r   r|   r  r/   r/   r0   r  h  s
   zJambaModel._update_mamba_mask)NNNNNNN)rB   rC   rD   r   r%   r   r   r   r'   r   rF   rH   rZ  rY  r   r   r   r=   r  rG   r/   r/   r-   r0   rp    s@    	
Crp  r2   gate_logitsr,  c                    s  | du s	t | tsdS t | tr#| d j tj fdd| D dd}tjjj|dd}tj||dd\}}tjj	||}|du rStj
| dd}	tj
|dd}
ng|j\}}|jd ||  }|dddddddf |||||fd|| }tj| | ddtj|dd }	|ddddddf ||||fd| }tj|| ddtj|dd }
t|	|
d }|| S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   c                    s   g | ]}|  qS r/   )r6   )rR   
layer_gatecompute_devicer/   r0   rV     s    z,load_balancing_loss_func.<locals>.<listcomp>rt   r3   )r  r>   rK   r'   rv   r   r   r   rJ  r4  r9   rE   r?   r   r   r6   r  r   )r  r,  rG  r   concatenated_gate_logitsrL  rS   selected_expertsr<  tokens_per_expertrouter_prob_per_expertrU   rN  ra   expert_attention_mask router_per_expert_attention_maskoverall_lossr/   r  r0   load_balancing_loss_funcv  s>   



r  c                       s   e Zd ZddiZddiZddgdgfiZdef fdd	Zee		
	
	
	
	
	
	
	
	
	dde
jd
B de
jd
B de
jd
B ded
B de
jd
B de
jd
B ded
B ded
B de
jd
B dee
jB dee defddZ  ZS )JambaForCausalLMzlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputr1   logitsrf   c                    sX   t  | t|| _|j| _tj|j|jdd| _|j	| _	|j
| _
|j| _|   d S r$  )r$   r%   rp  r]  rs  r   r   r,   r  router_aux_loss_coefr,  rF  r{  r)  r-   r/   r0   r%     s   
zJambaForCausalLM.__init__Nr   r~  r   rW  r   r  labelsr  output_router_logitsr|   logits_to_keepr   r#   c                 K   s   |dur|n| j j}| jd||||||||	d|}|j}t|
tr)t|
 dn|
}| |dd|ddf }d}|durK| j||| j	fi |}d}|rht
|j| j| j|}|durh|| j||j 7 }t||||j|j|j|jdS )aj  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, JambaForCausalLM

        >>> model = JambaForCausalLM.from_pretrained("ai21labs/Jamba-v0.1")
        >>> tokenizer = AutoTokenizer.from_pretrained("ai21labs/Jamba-v0.1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r~  r   rW  r   r  r  r  r|   )lossaux_lossr  r   r1   r_  rK  r/   )rf   r  r]  r  r  r   slicer  loss_functionrs  r  rK  r,  rF  r  r6   rK   r   r   r1   r_  )r+   r~  r   rW  r   r  r  r  r  r|   r  r   outputsr1   slice_indicesr  r  r  r/   r/   r0   r=     sN   (	zJambaForCausalLM.forward)
NNNNNNNNNr   )rB   rC   rD   _tied_weights_keys_tp_plan_pp_planr   r%   r   r   r'   r   rF   rH   rZ  rY  r   r   r   r   r=   rG   r/   r/   r-   r0   r    sT    	
r  c                   @   s   e Zd ZdS )JambaForSequenceClassificationN)rB   rC   rD   r/   r/   r/   r0   r  1  s    r  )r  r  rp  r\  )r   )r   )Nr2   N)Hcollections.abcr   typingr   r'   r    r   rb  activationsr   
generationr   integrationsr	   r
   r   r   r   masking_utilsr   modeling_layersr   r   modeling_outputsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.genericr   utils.output_capturingr   r   configuration_jambar   
get_loggerrB   r   Moduler    rH   r   r   rF   r   r   rE   r   r   r   r#  r*  rD  rO  r[  r\  rv  rp  r>   r  r  r  __all__r/   r/   r/   r0   <module>   s   
_
9   '%(!
i
Rh