o eiZ1@s&ddlZddlmZmZddlmZddlmZmZddl m Z ddl m Z ddl mZmZd d lmZd d lmZmZmZmZd d lmZmZeeZGd ddeZGdddeZGdddeZGdddeZ GdddeZ!Gddde Z"GdddeZ#GdddeZ$gdZ%dS)N)Cache DynamicCache)layer_type_validation)create_causal_mask!create_sliding_window_causal_mask)BaseModelOutputWithPast)Unpack)TransformersKwargslogging) LlamaConfig)LlamaDecoderLayerLlamaForCausalLM LlamaModelLlamaPreTrainedModel)Qwen2AttentionQwen2RotaryEmbeddingc,seZdZdZdZdZdddddd d d d d dddgdddddddddfdededededededededed ed!ed"e d#edBd$ed%e d&ed'ed(e d)e dBd*ed+e edBf*fd,d- Z Z S). CwmConfiga Configuration for Code World Model (CWM). This is an inherited Llama3-compatible configuration with layer-interleaved sliding-window attention. Configures a `CwmModel`. Designed to yield a configuration mirroring the model in the [facebook/cwm](https://huggingface.co/facebook/cwm) architecture by default. Other models include: - [facebook/cwm-sft](https://huggingface.co/facebook/cwm-sft) - [facebook/cwm-pretrain](https://huggingface.co/facebook/cwm-pretrain) Args: vocab_size (`int`, *optional*, defaults to 128256): Vocabulary size of the CWM model. Defines the number of different tokens that can be represented by the `inputs_ids` passed when calling [`CwmModel`] hidden_size (`int`, *optional*, defaults to 6144): Dimension of the hidden representations intermediate_size (`int`, *optional*, defaults to 21504): Dimension of the MLP representations num_hidden_layers (`int`, *optional*, defaults to 64): Number of hidden layers in the Transformer decoder num_attention_heads (`int`, *optional*, defaults to 48): Number of attention heads for each attention layer in the Transformer decoder num_key_value_heads (`int`, *optional*, defaults to 8): This is the number of key_value heads that should be used to implement Grouped Query Attention (GQA). If it is not specified, will default to `num_attention_heads`. head_dim (`int`, *optional*, defaults to 128): The attention head dimension. hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): The non-linear activation function (function or string) in the decoder. max_position_embeddings (`int`, *optional*, defaults to 131072): The maximum sequence length that this model might ever be used with. CWM's attention allows sequence lengths up to 131072 tokens. initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. rms_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the rms normalization layers. use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/values attentions (not used by all models). Only relevant if `config.is_decoder=True`. pad_token_id (`int`, *optional*): Padding token id. eos_token_id (`int` or `list[int]`, *optional*, defaults to `[128001, 128008, 128009]`): The id of the *end-of-sequence* token. Optionally, use a list to set multiple *end-of-sequence* tokens. bos_token_id (`int`, *optional*, defaults to 128000): The id of the *beginning-of-sequence* token. tie_word_embeddings (`bool`, *optional*, defaults to `False`): Whether to tie weight embeddings attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. pretraining_tp (`int`, *optional*, defaults to 1): Tensor parallelism degree used during pretraining. See [this document](https://huggingface.co/docs/transformers/parallelism) and [this issue](https://github.com/pytorch/pytorch/issues/76232). mlp_bias (`bool`, *optional*, defaults to `False`): Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers. rope_parameters (`RopeParameters`, *optional*): Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE with longer `max_position_embeddings`. sliding_window (`int`, *optional*, defaults to 8192): Sliding window attention window size. layer_types (`List[str]`, *optional*): List of layer types for each layer. Each element should be either "full_attention" or "sliding_attention". If not specified, will default to alternating pattern based on the provided window pattern. cwm.AiiiT@0siluig{Gz?gh㈵>TN)iii iFg vocab_size hidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_headshead_dim hidden_actmax_position_embeddingsinitializer_range rms_norm_eps use_cache pad_token_id bos_token_idtie_word_embeddingsattention_dropoutpretraining_tpmlp_biasrope_parameterssliding_window layer_typesc s|dur ddddddd}|durdfd d t|D}nt|||r*t|nd|_t||_tjd!id |d |d |d|d|d|d|d|d| d| d| d| d| dt|d|d|ddd|d|d|d |||`dS)"Nrg0@g@g?rllama3) rope_thetafactorhigh_freq_factorlow_freq_factor original_max_position_embeddings rope_typecs g|] }|dkr dndqS)rfull_attentionsliding_attention).0iwindow_patternr=a/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/cwm/modular_cwm.py sz&CwmConfig.__init__..rrr r!r"r#r$r%r&r'r(r)r* eos_token_idr+r,attention_biasFr-r0r.r/r=) rangerintr1listr2super__init__rE)selfrrr r!r"r#r$r%r&r'r(r)r*rDr+r,r-r.r/r0r1r2kwargs __class__r@rBrJisz          zCwmConfig.__init__)__name__ __module__ __qualname____doc__ model_type default_thetarGstrfloatbooldictrHrJ __classcell__r=r=rMrBr%s@      rc@ eZdZdS)CwmRotaryEmbeddingNrOrPrQr=r=r=rBr[r[c&eZdZdedeffdd ZZS) CwmAttentionconfig layer_idxcsntj||dtjj|j|j|jdd|_tjj|j|j |jdd|_ tjj|j|j |jdd|_ dS)Nr`raF)bias) rIrJtorchnnLinearrr"r$q_projr#k_projv_projrKr`rarMr=rBrJs"zCwmAttention.__init__rOrPrQrrGrJrYr=r=rMrBr_r_cr^)CwmDecoderLayerr`racs.tj||d|j||_t||d|_dS)Nrb)rIrJr2attention_typer_ self_attnrjrMr=rBrJs zCwmDecoderLayer.__init__rkr=r=rMrBrmrlrmc@rZ)CwmPreTrainedModelNr\r=r=r=rBrpr]rpc@rZ)CwmModelOutputWithPastNr\r=r=r=rBrqr]rqcseZdZeZdeffdd Z       ddejdBdejdBdejdBde dBd ej dBd ejdBd e dBd e e d efddZZS)CwmModelr`cs2ttjfddtjD|_dS)Ncsg|]}t|qSr=)rm)r>rar`r=rBrCsz%CwmModel.__init__..)rIrJrdre ModuleListrFr!layers)rKr`rMrsrBrJs  zCwmModel.__init__N input_idsattention_mask position_idspast_key_values inputs_embedscache_positionr)rLreturnc Ks2|du|duAr td|dur||}|r!|dur!t|jd}|dur<|dur-|nd} tj|jd|jd| }|durE| d}t |} t si|j|||||d} | } t d i| td i| d} |} || |}|jd|jjD]}|| f| |j||||d|} qz|| } t| |d S) Nz:You must specify exactly one of input_ids or inputs_embedsrsrr)device)r`rzrwr{ryrx)r;r<)rwrxryr{position_embeddings)last_hidden_stateryr=) ValueError embed_tokensrr`get_seq_lengthrdarangeshaper} unsqueeze isinstancerXcopyrr rotary_embrur!rnnormrq)rKrvrwrxryrzr{r)rLpast_seen_tokenscausal_mask_mapping mask_kwargssliding_mask_kwargs hidden_statesr~ decoder_layerr=r=rBforwardsV         zCwmModel.forward)NNNNNNN)rOrPrQr config_classrJrd LongTensorTensorr FloatTensorrWr r rqrrYr=r=rMrBrrs:  rrc@rZ)CwmForCausalLMNr\r=r=r=rBrr]r)rrprrr)&rd cache_utilsrrconfiguration_utilsr masking_utilsrrmodeling_outputsrprocessing_utilsr utilsr r llama.configuration_llamar llama.modeling_llamarrrrqwen2.modeling_qwen2rr get_loggerrOloggerrr[r_rmrprqrrr__all__r=r=r=rBs*     K