from __future__ import annotations import torch class GraphModule(torch.nn.Module): def forward(self, s72: "Sym(s72)", L_input_ids_: "i32[s72]", L_self_modules_embed_tokens_parameters_weight_: "bf16[166016, 896]", L_self_modules_layers_modules_0_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]", s80: "Sym(s72)", L_positions_: "i64[s72]", L_self_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_0_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_1_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_1_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_2_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_2_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_3_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_3_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_4_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_4_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_5_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_5_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_6_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_6_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_7_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_7_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_8_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_9_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_9_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_10_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_10_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_11_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_11_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_12_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_12_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_13_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_13_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_14_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_14_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_15_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_15_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_16_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_16_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_17_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_17_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_18_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_18_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_19_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_19_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_20_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_20_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_21_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_21_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_22_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_22_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_22_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_22_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_22_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_layers_modules_23_modules_input_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", L_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", L_self_modules_layers_modules_23_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", L_self_modules_layers_modules_23_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", L_self_modules_layers_modules_23_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", L_self_modules_layers_modules_23_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", L_self_modules_norm_parameters_weight_: "bf16[896]"): l_input_ids_ = L_input_ids_ l_self_modules_embed_tokens_parameters_weight_ = L_self_modules_embed_tokens_parameters_weight_ l_self_modules_layers_modules_0_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_0_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = L_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ l_positions_ = L_positions_ l_self_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_0_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_1_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_1_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_1_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_2_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_2_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_2_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_3_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_3_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_3_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_4_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_4_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_4_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_5_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_5_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_5_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_6_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_6_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_6_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_7_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_7_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_7_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_8_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_9_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_9_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_9_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_10_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_10_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_10_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_11_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_11_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_11_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_12_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_12_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_12_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_12_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_13_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_13_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_13_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_13_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_14_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_14_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_14_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_14_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_15_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_15_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_15_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_15_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_16_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_16_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_16_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_16_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_17_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_17_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_17_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_17_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_18_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_18_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_18_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_18_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_19_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_19_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_19_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_19_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_20_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_20_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_20_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_20_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_21_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_21_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_21_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_21_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_22_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_22_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_22_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_22_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_22_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_22_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_22_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_22_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_22_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_22_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_layers_modules_23_modules_input_layernorm_parameters_weight_ = L_self_modules_layers_modules_23_modules_input_layernorm_parameters_weight_ l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_bias_ = L_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_bias_ l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_weight_ = L_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_weight_ l_self_modules_layers_modules_23_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_layers_modules_23_modules_self_attn_modules_o_proj_parameters_weight_ l_self_modules_layers_modules_23_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_layers_modules_23_modules_post_attention_layernorm_parameters_weight_ l_self_modules_layers_modules_23_modules_mlp_modules_gate_up_proj_parameters_weight_ = L_self_modules_layers_modules_23_modules_mlp_modules_gate_up_proj_parameters_weight_ l_self_modules_layers_modules_23_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_layers_modules_23_modules_mlp_modules_down_proj_parameters_weight_ l_self_modules_norm_parameters_weight_ = L_self_modules_norm_parameters_weight_ # No stacktrace found for following nodes submod_0 = self.submod_0(l_input_ids_, s72, l_self_modules_embed_tokens_parameters_weight_, l_self_modules_layers_modules_0_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); l_input_ids_ = l_self_modules_embed_tokens_parameters_weight_ = l_self_modules_layers_modules_0_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem = submod_0[0] getitem_1 = submod_0[1] getitem_2 = submod_0[2] getitem_3 = submod_0[3] getitem_4 = submod_0[4]; submod_0 = None submod_1 = self.submod_1(getitem, s72, getitem_1, getitem_2, getitem_3); getitem = getitem_1 = getitem_2 = submod_1 = None submod_2 = self.submod_2(getitem_3, s72, l_self_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_, getitem_4, l_self_modules_layers_modules_0_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_1_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_3 = l_self_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_ = getitem_4 = l_self_modules_layers_modules_0_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_5 = submod_2[0] getitem_6 = submod_2[1] getitem_7 = submod_2[2] getitem_8 = submod_2[3] getitem_9 = submod_2[4]; submod_2 = None submod_3 = self.submod_3(getitem_5, s72, getitem_6, getitem_7, getitem_8); getitem_5 = getitem_6 = getitem_7 = submod_3 = None submod_4 = self.submod_4(getitem_8, s72, l_self_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_, getitem_9, l_self_modules_layers_modules_1_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_2_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_8 = l_self_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_ = getitem_9 = l_self_modules_layers_modules_1_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_10 = submod_4[0] getitem_11 = submod_4[1] getitem_12 = submod_4[2] getitem_13 = submod_4[3] getitem_14 = submod_4[4]; submod_4 = None submod_5 = self.submod_5(getitem_10, s72, getitem_11, getitem_12, getitem_13); getitem_10 = getitem_11 = getitem_12 = submod_5 = None submod_6 = self.submod_6(getitem_13, s72, l_self_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_, getitem_14, l_self_modules_layers_modules_2_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_3_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_13 = l_self_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_ = getitem_14 = l_self_modules_layers_modules_2_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_15 = submod_6[0] getitem_16 = submod_6[1] getitem_17 = submod_6[2] getitem_18 = submod_6[3] getitem_19 = submod_6[4]; submod_6 = None submod_7 = self.submod_7(getitem_15, s72, getitem_16, getitem_17, getitem_18); getitem_15 = getitem_16 = getitem_17 = submod_7 = None submod_8 = self.submod_8(getitem_18, s72, l_self_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_, getitem_19, l_self_modules_layers_modules_3_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_4_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_18 = l_self_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_ = getitem_19 = l_self_modules_layers_modules_3_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_20 = submod_8[0] getitem_21 = submod_8[1] getitem_22 = submod_8[2] getitem_23 = submod_8[3] getitem_24 = submod_8[4]; submod_8 = None submod_9 = self.submod_9(getitem_20, s72, getitem_21, getitem_22, getitem_23); getitem_20 = getitem_21 = getitem_22 = submod_9 = None submod_10 = self.submod_10(getitem_23, s72, l_self_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_, getitem_24, l_self_modules_layers_modules_4_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_5_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_23 = l_self_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_ = getitem_24 = l_self_modules_layers_modules_4_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_25 = submod_10[0] getitem_26 = submod_10[1] getitem_27 = submod_10[2] getitem_28 = submod_10[3] getitem_29 = submod_10[4]; submod_10 = None submod_11 = self.submod_11(getitem_25, s72, getitem_26, getitem_27, getitem_28); getitem_25 = getitem_26 = getitem_27 = submod_11 = None submod_12 = self.submod_12(getitem_28, s72, l_self_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_, getitem_29, l_self_modules_layers_modules_5_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_6_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_28 = l_self_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_ = getitem_29 = l_self_modules_layers_modules_5_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_30 = submod_12[0] getitem_31 = submod_12[1] getitem_32 = submod_12[2] getitem_33 = submod_12[3] getitem_34 = submod_12[4]; submod_12 = None submod_13 = self.submod_13(getitem_30, s72, getitem_31, getitem_32, getitem_33); getitem_30 = getitem_31 = getitem_32 = submod_13 = None submod_14 = self.submod_14(getitem_33, s72, l_self_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_, getitem_34, l_self_modules_layers_modules_6_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_7_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_33 = l_self_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_ = getitem_34 = l_self_modules_layers_modules_6_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_35 = submod_14[0] getitem_36 = submod_14[1] getitem_37 = submod_14[2] getitem_38 = submod_14[3] getitem_39 = submod_14[4]; submod_14 = None submod_15 = self.submod_15(getitem_35, s72, getitem_36, getitem_37, getitem_38); getitem_35 = getitem_36 = getitem_37 = submod_15 = None submod_16 = self.submod_16(getitem_38, s72, l_self_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_, getitem_39, l_self_modules_layers_modules_7_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_38 = l_self_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_ = getitem_39 = l_self_modules_layers_modules_7_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_40 = submod_16[0] getitem_41 = submod_16[1] getitem_42 = submod_16[2] getitem_43 = submod_16[3] getitem_44 = submod_16[4]; submod_16 = None submod_17 = self.submod_17(getitem_40, s72, getitem_41, getitem_42, getitem_43); getitem_40 = getitem_41 = getitem_42 = submod_17 = None submod_18 = self.submod_18(getitem_43, s72, l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_, getitem_44, l_self_modules_layers_modules_8_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_9_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_43 = l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ = getitem_44 = l_self_modules_layers_modules_8_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_45 = submod_18[0] getitem_46 = submod_18[1] getitem_47 = submod_18[2] getitem_48 = submod_18[3] getitem_49 = submod_18[4]; submod_18 = None submod_19 = self.submod_19(getitem_45, s72, getitem_46, getitem_47, getitem_48); getitem_45 = getitem_46 = getitem_47 = submod_19 = None submod_20 = self.submod_20(getitem_48, s72, l_self_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_, getitem_49, l_self_modules_layers_modules_9_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_10_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_48 = l_self_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_ = getitem_49 = l_self_modules_layers_modules_9_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_50 = submod_20[0] getitem_51 = submod_20[1] getitem_52 = submod_20[2] getitem_53 = submod_20[3] getitem_54 = submod_20[4]; submod_20 = None submod_21 = self.submod_21(getitem_50, s72, getitem_51, getitem_52, getitem_53); getitem_50 = getitem_51 = getitem_52 = submod_21 = None submod_22 = self.submod_22(getitem_53, s72, l_self_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_, getitem_54, l_self_modules_layers_modules_10_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_11_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_53 = l_self_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_ = getitem_54 = l_self_modules_layers_modules_10_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_55 = submod_22[0] getitem_56 = submod_22[1] getitem_57 = submod_22[2] getitem_58 = submod_22[3] getitem_59 = submod_22[4]; submod_22 = None submod_23 = self.submod_23(getitem_55, s72, getitem_56, getitem_57, getitem_58); getitem_55 = getitem_56 = getitem_57 = submod_23 = None submod_24 = self.submod_24(getitem_58, s72, l_self_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_, getitem_59, l_self_modules_layers_modules_11_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_12_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_58 = l_self_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_ = getitem_59 = l_self_modules_layers_modules_11_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_12_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_60 = submod_24[0] getitem_61 = submod_24[1] getitem_62 = submod_24[2] getitem_63 = submod_24[3] getitem_64 = submod_24[4]; submod_24 = None submod_25 = self.submod_25(getitem_60, s72, getitem_61, getitem_62, getitem_63); getitem_60 = getitem_61 = getitem_62 = submod_25 = None submod_26 = self.submod_26(getitem_63, s72, l_self_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_, getitem_64, l_self_modules_layers_modules_12_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_13_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_63 = l_self_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_ = getitem_64 = l_self_modules_layers_modules_12_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_13_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_65 = submod_26[0] getitem_66 = submod_26[1] getitem_67 = submod_26[2] getitem_68 = submod_26[3] getitem_69 = submod_26[4]; submod_26 = None submod_27 = self.submod_27(getitem_65, s72, getitem_66, getitem_67, getitem_68); getitem_65 = getitem_66 = getitem_67 = submod_27 = None submod_28 = self.submod_28(getitem_68, s72, l_self_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_, getitem_69, l_self_modules_layers_modules_13_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_14_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_68 = l_self_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_ = getitem_69 = l_self_modules_layers_modules_13_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_14_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_70 = submod_28[0] getitem_71 = submod_28[1] getitem_72 = submod_28[2] getitem_73 = submod_28[3] getitem_74 = submod_28[4]; submod_28 = None submod_29 = self.submod_29(getitem_70, s72, getitem_71, getitem_72, getitem_73); getitem_70 = getitem_71 = getitem_72 = submod_29 = None submod_30 = self.submod_30(getitem_73, s72, l_self_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_, getitem_74, l_self_modules_layers_modules_14_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_15_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_73 = l_self_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_ = getitem_74 = l_self_modules_layers_modules_14_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_15_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_75 = submod_30[0] getitem_76 = submod_30[1] getitem_77 = submod_30[2] getitem_78 = submod_30[3] getitem_79 = submod_30[4]; submod_30 = None submod_31 = self.submod_31(getitem_75, s72, getitem_76, getitem_77, getitem_78); getitem_75 = getitem_76 = getitem_77 = submod_31 = None submod_32 = self.submod_32(getitem_78, s72, l_self_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_, getitem_79, l_self_modules_layers_modules_15_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_16_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_78 = l_self_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_ = getitem_79 = l_self_modules_layers_modules_15_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_16_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_80 = submod_32[0] getitem_81 = submod_32[1] getitem_82 = submod_32[2] getitem_83 = submod_32[3] getitem_84 = submod_32[4]; submod_32 = None submod_33 = self.submod_33(getitem_80, s72, getitem_81, getitem_82, getitem_83); getitem_80 = getitem_81 = getitem_82 = submod_33 = None submod_34 = self.submod_34(getitem_83, s72, l_self_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_, getitem_84, l_self_modules_layers_modules_16_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_17_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_83 = l_self_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_ = getitem_84 = l_self_modules_layers_modules_16_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_17_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_85 = submod_34[0] getitem_86 = submod_34[1] getitem_87 = submod_34[2] getitem_88 = submod_34[3] getitem_89 = submod_34[4]; submod_34 = None submod_35 = self.submod_35(getitem_85, s72, getitem_86, getitem_87, getitem_88); getitem_85 = getitem_86 = getitem_87 = submod_35 = None submod_36 = self.submod_36(getitem_88, s72, l_self_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_, getitem_89, l_self_modules_layers_modules_17_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_18_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_88 = l_self_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_ = getitem_89 = l_self_modules_layers_modules_17_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_18_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_90 = submod_36[0] getitem_91 = submod_36[1] getitem_92 = submod_36[2] getitem_93 = submod_36[3] getitem_94 = submod_36[4]; submod_36 = None submod_37 = self.submod_37(getitem_90, s72, getitem_91, getitem_92, getitem_93); getitem_90 = getitem_91 = getitem_92 = submod_37 = None submod_38 = self.submod_38(getitem_93, s72, l_self_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_, getitem_94, l_self_modules_layers_modules_18_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_19_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_93 = l_self_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_ = getitem_94 = l_self_modules_layers_modules_18_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_19_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_95 = submod_38[0] getitem_96 = submod_38[1] getitem_97 = submod_38[2] getitem_98 = submod_38[3] getitem_99 = submod_38[4]; submod_38 = None submod_39 = self.submod_39(getitem_95, s72, getitem_96, getitem_97, getitem_98); getitem_95 = getitem_96 = getitem_97 = submod_39 = None submod_40 = self.submod_40(getitem_98, s72, l_self_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_, getitem_99, l_self_modules_layers_modules_19_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_20_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_98 = l_self_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_ = getitem_99 = l_self_modules_layers_modules_19_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_20_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_100 = submod_40[0] getitem_101 = submod_40[1] getitem_102 = submod_40[2] getitem_103 = submod_40[3] getitem_104 = submod_40[4]; submod_40 = None submod_41 = self.submod_41(getitem_100, s72, getitem_101, getitem_102, getitem_103); getitem_100 = getitem_101 = getitem_102 = submod_41 = None submod_42 = self.submod_42(getitem_103, s72, l_self_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_, getitem_104, l_self_modules_layers_modules_20_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_21_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_103 = l_self_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_ = getitem_104 = l_self_modules_layers_modules_20_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_21_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_105 = submod_42[0] getitem_106 = submod_42[1] getitem_107 = submod_42[2] getitem_108 = submod_42[3] getitem_109 = submod_42[4]; submod_42 = None submod_43 = self.submod_43(getitem_105, s72, getitem_106, getitem_107, getitem_108); getitem_105 = getitem_106 = getitem_107 = submod_43 = None submod_44 = self.submod_44(getitem_108, s72, l_self_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_, getitem_109, l_self_modules_layers_modules_21_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_22_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_108 = l_self_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_ = getitem_109 = l_self_modules_layers_modules_21_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_22_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_bias_ = None getitem_110 = submod_44[0] getitem_111 = submod_44[1] getitem_112 = submod_44[2] getitem_113 = submod_44[3] getitem_114 = submod_44[4]; submod_44 = None submod_45 = self.submod_45(getitem_110, s72, getitem_111, getitem_112, getitem_113); getitem_110 = getitem_111 = getitem_112 = submod_45 = None submod_46 = self.submod_46(getitem_113, s72, l_self_modules_layers_modules_22_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_22_modules_post_attention_layernorm_parameters_weight_, getitem_114, l_self_modules_layers_modules_22_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_22_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_layers_modules_23_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_bias_, l_positions_, l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_); getitem_113 = l_self_modules_layers_modules_22_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_22_modules_post_attention_layernorm_parameters_weight_ = getitem_114 = l_self_modules_layers_modules_22_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_22_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_layers_modules_23_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_bias_ = l_positions_ = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = None getitem_115 = submod_46[0] getitem_116 = submod_46[1] getitem_117 = submod_46[2] getitem_118 = submod_46[3] getitem_119 = submod_46[4]; submod_46 = None submod_47 = self.submod_47(getitem_115, s72, getitem_116, getitem_117, getitem_118); getitem_115 = getitem_116 = getitem_117 = submod_47 = None submod_48 = self.submod_48(getitem_118, s72, l_self_modules_layers_modules_23_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_23_modules_post_attention_layernorm_parameters_weight_, getitem_119, l_self_modules_layers_modules_23_modules_mlp_modules_gate_up_proj_parameters_weight_, l_self_modules_layers_modules_23_modules_mlp_modules_down_proj_parameters_weight_, l_self_modules_norm_parameters_weight_); getitem_118 = s72 = l_self_modules_layers_modules_23_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_23_modules_post_attention_layernorm_parameters_weight_ = getitem_119 = l_self_modules_layers_modules_23_modules_mlp_modules_gate_up_proj_parameters_weight_ = l_self_modules_layers_modules_23_modules_mlp_modules_down_proj_parameters_weight_ = l_self_modules_norm_parameters_weight_ = None return (submod_48,) class submod_0(torch.nn.Module): def forward(self, l_input_ids_: "i32[s72]", s72: "Sym(s72)", l_self_modules_embed_tokens_parameters_weight_: "bf16[166016, 896]", l_self_modules_layers_modules_0_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/vocab_parallel_embedding.py:478 in forward_native, code: output_parallel = self.quant_method.embedding(self, masked_input.long()) long: "i64[s72]" = l_input_ids_.long(); l_input_ids_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/vocab_parallel_embedding.py:72 in embedding, code: return F.embedding(input_, layer.weight) embedding: "bf16[s72, 896]" = torch.nn.functional.embedding(long, l_self_modules_embed_tokens_parameters_weight_); long = l_self_modules_embed_tokens_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_0_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_0_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = embedding.to(torch.float32) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = to.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add); add = None mul: "f32[s72, 896]" = to * rsqrt; to = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_1: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_1 * _get_data_attr; to_1 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 1152]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_1 = l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear.split([896, 128, 128], dim = -1); linear = None getitem: "bf16[s72, 896]" = split[0] getitem_1: "bf16[s72, 128]" = split[1] getitem_2: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_3: "bf16[s72, 32]" = chunk[0] getitem_4: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view: "bf16[s72, 14, 64]" = getitem.view(s72, -1, 64); getitem = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_5: "bf16[s72, 14, 64]" = view[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_6: "bf16[s72, 14, 0]" = view[(Ellipsis, slice(64, None, None))]; view = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_3.unsqueeze(-2) to_2: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_4.unsqueeze(-2) to_3: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_5, 2, dim = -1); getitem_5 = None getitem_7: "bf16[s72, 14, 32]" = chunk_1[0] getitem_8: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_2: "bf16[s72, 14, 32]" = getitem_7 * to_2 mul_3: "bf16[s72, 14, 32]" = getitem_8 * to_3 sub: "bf16[s72, 14, 32]" = mul_2 - mul_3; mul_2 = mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_4: "bf16[s72, 14, 32]" = getitem_8 * to_2; getitem_8 = to_2 = None mul_5: "bf16[s72, 14, 32]" = getitem_7 * to_3; getitem_7 = to_3 = None add_1: "bf16[s72, 14, 32]" = mul_4 + mul_5; mul_4 = mul_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_1), dim = -1); sub = add_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_6), dim = -1); cat = getitem_6 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_1.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_1: "bf16[s72, 2, 64]" = getitem_1.view(s72, -1, 64); getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_9: "bf16[s72, 2, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_10: "bf16[s72, 2, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_3.unsqueeze(-2); getitem_3 = None to_4: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_4.unsqueeze(-2); getitem_4 = None to_5: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_9, 2, dim = -1); getitem_9 = None getitem_11: "bf16[s72, 2, 32]" = chunk_2[0] getitem_12: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_6: "bf16[s72, 2, 32]" = getitem_11 * to_4 mul_7: "bf16[s72, 2, 32]" = getitem_12 * to_5 sub_1: "bf16[s72, 2, 32]" = mul_6 - mul_7; mul_6 = mul_7 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_8: "bf16[s72, 2, 32]" = getitem_12 * to_4; getitem_12 = to_4 = None mul_9: "bf16[s72, 2, 32]" = getitem_11 * to_5; getitem_11 = to_5 = None add_2: "bf16[s72, 2, 32]" = mul_8 + mul_9; mul_8 = mul_9 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_2), dim = -1); sub_1 = add_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_10), dim = -1); cat_2 = getitem_10 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_2: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_3: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_4: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_5: "bf16[s72, 2, 64]" = getitem_2.view(-1, 2, 64); getitem_2 = None return (view_4, view_5, view_2, view_3, embedding) class submod_1(torch.nn.Module): def forward(self, key_2: "bf16[s72, 2, 64]", s72: "Sym(s72)", value: "bf16[s72, 2, 64]", query_2: "bf16[s72, 14, 64]", output_3: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_2, value, 'model.layers.0.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_2, key_2, value, output_3, 'model.layers.0.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_2 = key_2 = value = output_3 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_2(torch.nn.Module): def forward(self, output_3: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", output_parallel: "bf16[s72, 896]", l_self_modules_layers_modules_0_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_1_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_3.view(-1, 896); output_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + output_parallel; to = output_parallel = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_0_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_0_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_1_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_1_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_3(torch.nn.Module): def forward(self, key_5: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_1: "bf16[s72, 2, 64]", query_5: "bf16[s72, 14, 64]", output_7: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_5, value_1, 'model.layers.1.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_5, key_5, value_1, output_7, 'model.layers.1.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_5 = key_5 = value_1 = output_7 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_4(torch.nn.Module): def forward(self, output_7: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_1: "bf16[s72, 896]", l_self_modules_layers_modules_1_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_2_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_7.view(-1, 896); output_7 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_1; to = residual_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_1_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_1_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_2_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_2_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_5(torch.nn.Module): def forward(self, key_8: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_2: "bf16[s72, 2, 64]", query_8: "bf16[s72, 14, 64]", output_11: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_8, value_2, 'model.layers.2.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_2, output_11, 'model.layers.2.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_8 = key_8 = value_2 = output_11 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_6(torch.nn.Module): def forward(self, output_11: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_3: "bf16[s72, 896]", l_self_modules_layers_modules_2_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_3_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_11.view(-1, 896); output_11 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_3; to = residual_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_2_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_2_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_3_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_3_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_7(torch.nn.Module): def forward(self, key_11: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_3: "bf16[s72, 2, 64]", query_11: "bf16[s72, 14, 64]", output_15: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_11, value_3, 'model.layers.3.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_11, key_11, value_3, output_15, 'model.layers.3.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_11 = key_11 = value_3 = output_15 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_8(torch.nn.Module): def forward(self, output_15: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_5: "bf16[s72, 896]", l_self_modules_layers_modules_3_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_4_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_15.view(-1, 896); output_15 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_5; to = residual_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_3_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_3_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_4_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_4_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_9(torch.nn.Module): def forward(self, key_14: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_4: "bf16[s72, 2, 64]", query_14: "bf16[s72, 14, 64]", output_19: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_14, value_4, 'model.layers.4.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_14, key_14, value_4, output_19, 'model.layers.4.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_14 = key_14 = value_4 = output_19 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_10(torch.nn.Module): def forward(self, output_19: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_7: "bf16[s72, 896]", l_self_modules_layers_modules_4_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_5_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_19.view(-1, 896); output_19 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_7; to = residual_7 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_4_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_4_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_5_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_5_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_11(torch.nn.Module): def forward(self, key_17: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_5: "bf16[s72, 2, 64]", query_17: "bf16[s72, 14, 64]", output_23: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_17, value_5, 'model.layers.5.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_17, key_17, value_5, output_23, 'model.layers.5.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_17 = key_17 = value_5 = output_23 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_12(torch.nn.Module): def forward(self, output_23: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_9: "bf16[s72, 896]", l_self_modules_layers_modules_5_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_6_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_23.view(-1, 896); output_23 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_9; to = residual_9 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_5_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_5_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_6_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_6_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_13(torch.nn.Module): def forward(self, key_20: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_6: "bf16[s72, 2, 64]", query_20: "bf16[s72, 14, 64]", output_27: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_20, value_6, 'model.layers.6.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_20, key_20, value_6, output_27, 'model.layers.6.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_20 = key_20 = value_6 = output_27 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_14(torch.nn.Module): def forward(self, output_27: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_11: "bf16[s72, 896]", l_self_modules_layers_modules_6_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_7_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_27.view(-1, 896); output_27 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_11; to = residual_11 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_6_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_6_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_7_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_7_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_15(torch.nn.Module): def forward(self, key_23: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_7: "bf16[s72, 2, 64]", query_23: "bf16[s72, 14, 64]", output_31: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_23, value_7, 'model.layers.7.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_23, key_23, value_7, output_31, 'model.layers.7.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_23 = key_23 = value_7 = output_31 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_16(torch.nn.Module): def forward(self, output_31: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_13: "bf16[s72, 896]", l_self_modules_layers_modules_7_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_31.view(-1, 896); output_31 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_13; to = residual_13 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_7_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_7_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_17(torch.nn.Module): def forward(self, key_26: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_8: "bf16[s72, 2, 64]", query_26: "bf16[s72, 14, 64]", output_35: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_26, value_8, 'model.layers.8.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_26, key_26, value_8, output_35, 'model.layers.8.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_26 = key_26 = value_8 = output_35 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_18(torch.nn.Module): def forward(self, output_35: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_15: "bf16[s72, 896]", l_self_modules_layers_modules_8_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_9_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_35.view(-1, 896); output_35 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_15; to = residual_15 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_8_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_8_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_9_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_9_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_19(torch.nn.Module): def forward(self, key_29: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_9: "bf16[s72, 2, 64]", query_29: "bf16[s72, 14, 64]", output_39: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_29, value_9, 'model.layers.9.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_29, key_29, value_9, output_39, 'model.layers.9.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_29 = key_29 = value_9 = output_39 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_20(torch.nn.Module): def forward(self, output_39: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_17: "bf16[s72, 896]", l_self_modules_layers_modules_9_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_10_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_39.view(-1, 896); output_39 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_17; to = residual_17 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_9_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_9_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_10_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_10_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_21(torch.nn.Module): def forward(self, key_32: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_10: "bf16[s72, 2, 64]", query_32: "bf16[s72, 14, 64]", output_43: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_32, value_10, 'model.layers.10.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_32, key_32, value_10, output_43, 'model.layers.10.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_32 = key_32 = value_10 = output_43 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_22(torch.nn.Module): def forward(self, output_43: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_19: "bf16[s72, 896]", l_self_modules_layers_modules_10_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_11_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_43.view(-1, 896); output_43 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_19; to = residual_19 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_10_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_10_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_11_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_11_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_23(torch.nn.Module): def forward(self, key_35: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_11: "bf16[s72, 2, 64]", query_35: "bf16[s72, 14, 64]", output_47: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_35, value_11, 'model.layers.11.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_35, key_35, value_11, output_47, 'model.layers.11.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_35 = key_35 = value_11 = output_47 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_24(torch.nn.Module): def forward(self, output_47: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_21: "bf16[s72, 896]", l_self_modules_layers_modules_11_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_12_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_47.view(-1, 896); output_47 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_21; to = residual_21 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_11_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_11_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_12_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_12_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_12_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_25(torch.nn.Module): def forward(self, key_38: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_12: "bf16[s72, 2, 64]", query_38: "bf16[s72, 14, 64]", output_51: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_38, value_12, 'model.layers.12.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_38, key_38, value_12, output_51, 'model.layers.12.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_38 = key_38 = value_12 = output_51 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_26(torch.nn.Module): def forward(self, output_51: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_23: "bf16[s72, 896]", l_self_modules_layers_modules_12_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_13_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_51.view(-1, 896); output_51 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_23; to = residual_23 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_12_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_12_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_13_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_13_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_13_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_27(torch.nn.Module): def forward(self, key_41: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_13: "bf16[s72, 2, 64]", query_41: "bf16[s72, 14, 64]", output_55: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_41, value_13, 'model.layers.13.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_41, key_41, value_13, output_55, 'model.layers.13.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_41 = key_41 = value_13 = output_55 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_28(torch.nn.Module): def forward(self, output_55: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_25: "bf16[s72, 896]", l_self_modules_layers_modules_13_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_14_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_55.view(-1, 896); output_55 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_25; to = residual_25 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_13_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_13_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_14_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_14_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_14_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_29(torch.nn.Module): def forward(self, key_44: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_14: "bf16[s72, 2, 64]", query_44: "bf16[s72, 14, 64]", output_59: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_44, value_14, 'model.layers.14.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_44, key_44, value_14, output_59, 'model.layers.14.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_44 = key_44 = value_14 = output_59 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_30(torch.nn.Module): def forward(self, output_59: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_27: "bf16[s72, 896]", l_self_modules_layers_modules_14_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_15_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_59.view(-1, 896); output_59 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_27; to = residual_27 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_14_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_14_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_15_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_15_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_15_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_31(torch.nn.Module): def forward(self, key_47: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_15: "bf16[s72, 2, 64]", query_47: "bf16[s72, 14, 64]", output_63: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_47, value_15, 'model.layers.15.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_47, key_47, value_15, output_63, 'model.layers.15.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_47 = key_47 = value_15 = output_63 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_32(torch.nn.Module): def forward(self, output_63: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_29: "bf16[s72, 896]", l_self_modules_layers_modules_15_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_16_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_63.view(-1, 896); output_63 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_29; to = residual_29 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_15_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_15_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_16_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_16_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_16_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_33(torch.nn.Module): def forward(self, key_50: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_16: "bf16[s72, 2, 64]", query_50: "bf16[s72, 14, 64]", output_67: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_50, value_16, 'model.layers.16.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_50, key_50, value_16, output_67, 'model.layers.16.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_50 = key_50 = value_16 = output_67 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_34(torch.nn.Module): def forward(self, output_67: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_31: "bf16[s72, 896]", l_self_modules_layers_modules_16_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_17_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_67.view(-1, 896); output_67 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_31; to = residual_31 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_16_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_16_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_17_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_17_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_17_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_35(torch.nn.Module): def forward(self, key_53: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_17: "bf16[s72, 2, 64]", query_53: "bf16[s72, 14, 64]", output_71: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_53, value_17, 'model.layers.17.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_53, key_53, value_17, output_71, 'model.layers.17.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_53 = key_53 = value_17 = output_71 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_36(torch.nn.Module): def forward(self, output_71: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_33: "bf16[s72, 896]", l_self_modules_layers_modules_17_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_18_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_71.view(-1, 896); output_71 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_33; to = residual_33 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_17_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_17_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_18_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_18_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_18_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_37(torch.nn.Module): def forward(self, key_56: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_18: "bf16[s72, 2, 64]", query_56: "bf16[s72, 14, 64]", output_75: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_56, value_18, 'model.layers.18.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_56, key_56, value_18, output_75, 'model.layers.18.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_56 = key_56 = value_18 = output_75 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_38(torch.nn.Module): def forward(self, output_75: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_35: "bf16[s72, 896]", l_self_modules_layers_modules_18_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_19_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_75.view(-1, 896); output_75 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_35; to = residual_35 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_18_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_18_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_19_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_19_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_19_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_39(torch.nn.Module): def forward(self, key_59: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_19: "bf16[s72, 2, 64]", query_59: "bf16[s72, 14, 64]", output_79: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_59, value_19, 'model.layers.19.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_59, key_59, value_19, output_79, 'model.layers.19.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_59 = key_59 = value_19 = output_79 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_40(torch.nn.Module): def forward(self, output_79: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_37: "bf16[s72, 896]", l_self_modules_layers_modules_19_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_20_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_79.view(-1, 896); output_79 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_37; to = residual_37 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_19_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_19_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_20_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_20_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_20_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_41(torch.nn.Module): def forward(self, key_62: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_20: "bf16[s72, 2, 64]", query_62: "bf16[s72, 14, 64]", output_83: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_62, value_20, 'model.layers.20.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_62, key_62, value_20, output_83, 'model.layers.20.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_62 = key_62 = value_20 = output_83 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_42(torch.nn.Module): def forward(self, output_83: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_39: "bf16[s72, 896]", l_self_modules_layers_modules_20_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_21_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_83.view(-1, 896); output_83 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_39; to = residual_39 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_20_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_20_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_21_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_21_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_21_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_43(torch.nn.Module): def forward(self, key_65: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_21: "bf16[s72, 2, 64]", query_65: "bf16[s72, 14, 64]", output_87: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_65, value_21, 'model.layers.21.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_65, key_65, value_21, output_87, 'model.layers.21.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_65 = key_65 = value_21 = output_87 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_44(torch.nn.Module): def forward(self, output_87: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_41: "bf16[s72, 896]", l_self_modules_layers_modules_21_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_22_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_87.view(-1, 896); output_87 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_41; to = residual_41 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_21_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_21_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_22_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_22_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_22_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_45(torch.nn.Module): def forward(self, key_68: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_22: "bf16[s72, 2, 64]", query_68: "bf16[s72, 14, 64]", output_91: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_68, value_22, 'model.layers.22.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_68, key_68, value_22, output_91, 'model.layers.22.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_68 = key_68 = value_22 = output_91 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_46(torch.nn.Module): def forward(self, output_91: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_22_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_22_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_43: "bf16[s72, 896]", l_self_modules_layers_modules_22_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_22_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_layers_modules_23_modules_input_layernorm_parameters_weight_: "bf16[896]", l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_weight_: "bf16[1152, 896]", l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_bias_: "bf16[1152]", l_positions_: "i64[s72]", l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_: "bf16[32768, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_91.view(-1, 896); output_91 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_22_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_22_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_22_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_22_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_43; to = residual_43 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_22_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_22_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_22_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_22_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_23_modules_input_layernorm_parameters_weight_); l_self_modules_layers_modules_23_modules_input_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_3: "bf16[s72, 1152]" = torch._C._nn.linear(mul_4, l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_weight_, l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_bias_); mul_4 = l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_weight_ = l_self_modules_layers_modules_23_modules_self_attn_modules_qkv_proj_parameters_bias_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py:209 in forward, code: q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) split = linear_3.split([896, 128, 128], dim = -1); linear_3 = None getitem_2: "bf16[s72, 896]" = split[0] getitem_3: "bf16[s72, 128]" = split[1] getitem_4: "bf16[s72, 128]" = split[2]; split = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:127 in forward_static, code: positions = positions.flatten() flatten: "i64[s72]" = l_positions_.flatten(); l_positions_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:129 in forward_static, code: cos_sin = cos_sin_cache.index_select(0, positions) index_select: "bf16[s72, 64]" = l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_.index_select(0, flatten); l_self_modules_layers_modules_0_modules_self_attn_modules_rotary_emb_buffers_cos_sin_cache_ = flatten = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:130 in forward_static, code: cos, sin = cos_sin.chunk(2, dim=-1) chunk = index_select.chunk(2, dim = -1); index_select = None getitem_5: "bf16[s72, 32]" = chunk[0] getitem_6: "bf16[s72, 32]" = chunk[1]; chunk = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:132 in forward_static, code: query_shape = query.shape size = getitem_2.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:133 in forward_static, code: query = query.view(num_tokens, -1, head_size) view_1: "bf16[s72, 14, 64]" = getitem_2.view(s72, -1, 64); getitem_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:134 in forward_static, code: query_rot = query[..., :rotary_dim] getitem_7: "bf16[s72, 14, 64]" = view_1[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:135 in forward_static, code: query_pass = query[..., rotary_dim:] getitem_8: "bf16[s72, 14, 0]" = view_1[(Ellipsis, slice(64, None, None))]; view_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2) to_6: "bf16[s72, 1, 32]" = unsqueeze.to(torch.bfloat16); unsqueeze = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_1: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2) to_7: "bf16[s72, 1, 32]" = unsqueeze_1.to(torch.bfloat16); unsqueeze_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_1 = torch.chunk(getitem_7, 2, dim = -1); getitem_7 = None getitem_9: "bf16[s72, 14, 32]" = chunk_1[0] getitem_10: "bf16[s72, 14, 32]" = chunk_1[1]; chunk_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_5: "bf16[s72, 14, 32]" = getitem_9 * to_6 mul_6: "bf16[s72, 14, 32]" = getitem_10 * to_7 sub: "bf16[s72, 14, 32]" = mul_5 - mul_6; mul_5 = mul_6 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_7: "bf16[s72, 14, 32]" = getitem_10 * to_6; getitem_10 = to_6 = None mul_8: "bf16[s72, 14, 32]" = getitem_9 * to_7; getitem_9 = to_7 = None add_4: "bf16[s72, 14, 32]" = mul_7 + mul_8; mul_7 = mul_8 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat: "bf16[s72, 14, 64]" = torch.cat((sub, add_4), dim = -1); sub = add_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:142 in forward_static, code: query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) cat_1: "bf16[s72, 14, 64]" = torch.cat((cat, getitem_8), dim = -1); cat = getitem_8 = None reshape: "bf16[s72, 896]" = cat_1.reshape(size); cat_1 = size = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:146 in forward_static, code: key_shape = key.shape size_1 = getitem_3.size() # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:147 in forward_static, code: key = key.view(num_tokens, -1, head_size) view_2: "bf16[s72, 2, 64]" = getitem_3.view(s72, -1, 64); getitem_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:148 in forward_static, code: key_rot = key[..., :rotary_dim] getitem_11: "bf16[s72, 2, 64]" = view_2[(Ellipsis, slice(None, 64, None))] # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:149 in forward_static, code: key_pass = key[..., rotary_dim:] getitem_12: "bf16[s72, 2, 0]" = view_2[(Ellipsis, slice(64, None, None))]; view_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:163 in forward_static, code: cos = cos.unsqueeze(-2).to(x.dtype) unsqueeze_2: "bf16[s72, 1, 32]" = getitem_5.unsqueeze(-2); getitem_5 = None to_8: "bf16[s72, 1, 32]" = unsqueeze_2.to(torch.bfloat16); unsqueeze_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:164 in forward_static, code: sin = sin.unsqueeze(-2).to(x.dtype) unsqueeze_3: "bf16[s72, 1, 32]" = getitem_6.unsqueeze(-2); getitem_6 = None to_9: "bf16[s72, 1, 32]" = unsqueeze_3.to(torch.bfloat16); unsqueeze_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:167 in forward_static, code: x1, x2 = torch.chunk(x, 2, dim=-1) chunk_2 = torch.chunk(getitem_11, 2, dim = -1); getitem_11 = None getitem_13: "bf16[s72, 2, 32]" = chunk_2[0] getitem_14: "bf16[s72, 2, 32]" = chunk_2[1]; chunk_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:172 in forward_static, code: o1 = x1 * cos - x2 * sin mul_9: "bf16[s72, 2, 32]" = getitem_13 * to_8 mul_10: "bf16[s72, 2, 32]" = getitem_14 * to_9 sub_1: "bf16[s72, 2, 32]" = mul_9 - mul_10; mul_9 = mul_10 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:173 in forward_static, code: o2 = x2 * cos + x1 * sin mul_11: "bf16[s72, 2, 32]" = getitem_14 * to_8; getitem_14 = to_8 = None mul_12: "bf16[s72, 2, 32]" = getitem_13 * to_9; getitem_13 = to_9 = None add_5: "bf16[s72, 2, 32]" = mul_11 + mul_12; mul_11 = mul_12 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/common.py:176 in forward_static, code: output = torch.cat((o1, o2), dim=-1) cat_2: "bf16[s72, 2, 64]" = torch.cat((sub_1, add_5), dim = -1); sub_1 = add_5 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding/base.py:156 in forward_static, code: key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) cat_3: "bf16[s72, 2, 64]" = torch.cat((cat_2, getitem_12), dim = -1); cat_2 = getitem_12 = None reshape_1: "bf16[s72, 128]" = cat_3.reshape(size_1); cat_3 = size_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:381 in forward, code: output = torch.empty(output_shape, dtype=output_dtype, device=query.device) size_2 = torch.Size([s72, 896]); s72 = None empty: "bf16[s72, 896]" = torch.empty(size_2, dtype = torch.bfloat16, device = device(type='cuda', index=0)); size_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:386 in forward, code: query = query.view(-1, self.num_heads, self.head_size) view_3: "bf16[s72, 14, 64]" = reshape.view(-1, 14, 64); reshape = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:387 in forward, code: output = output.view(-1, self.num_heads, self.head_size_v) view_4: "bf16[s72, 14, 64]" = empty.view(-1, 14, 64); empty = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:389 in forward, code: key = key.view(-1, self.num_kv_heads, self.head_size) view_5: "bf16[s72, 2, 64]" = reshape_1.view(-1, 2, 64); reshape_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:391 in forward, code: value = value.view(-1, self.num_kv_heads, self.head_size_v) view_6: "bf16[s72, 2, 64]" = getitem_4.view(-1, 2, 64); getitem_4 = None return (view_5, view_6, view_3, view_4, to_4) class submod_47(torch.nn.Module): def forward(self, key_71: "bf16[s72, 2, 64]", s72: "Sym(s72)", value_23: "bf16[s72, 2, 64]", query_71: "bf16[s72, 14, 64]", output_95: "bf16[s72, 14, 64]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:412 in forward, code: kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( unified_kv_cache_update: "bf16[0]" = torch.ops.vllm.unified_kv_cache_update(key_71, value_23, 'model.layers.23.self_attn.attn') # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:415 in forward, code: torch.ops.vllm.unified_attention_with_output( unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_71, key_71, value_23, output_95, 'model.layers.23.self_attn.attn', kv_cache_dummy_dep = unified_kv_cache_update); query_71 = key_71 = value_23 = output_95 = unified_kv_cache_update = unified_attention_with_output = None return () class submod_48(torch.nn.Module): def forward(self, output_95: "bf16[s72, 14, 64]", s72: "Sym(s72)", l_self_modules_layers_modules_23_modules_self_attn_modules_o_proj_parameters_weight_: "bf16[896, 896]", l_self_modules_layers_modules_23_modules_post_attention_layernorm_parameters_weight_: "bf16[896]", residual_45: "bf16[s72, 896]", l_self_modules_layers_modules_23_modules_mlp_modules_gate_up_proj_parameters_weight_: "bf16[9728, 896]", l_self_modules_layers_modules_23_modules_mlp_modules_down_proj_parameters_weight_: "bf16[896, 4864]", l_self_modules_norm_parameters_weight_: "bf16[896]"): # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/attention/layer.py:423 in forward, code: return output.view(-1, hidden_size) view: "bf16[s72, 896]" = output_95.view(-1, 896); output_95 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear: "bf16[s72, 896]" = torch._C._nn.linear(view, l_self_modules_layers_modules_23_modules_self_attn_modules_o_proj_parameters_weight_, None); view = l_self_modules_layers_modules_23_modules_self_attn_modules_o_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_layers_modules_23_modules_post_attention_layernorm_parameters_weight_); l_self_modules_layers_modules_23_modules_post_attention_layernorm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to: "f32[s72, 896]" = linear.to(torch.float32); linear = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add: "f32[s72, 896]" = to + residual_45; to = residual_45 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_1: "bf16[s72, 896]" = add.to(torch.bfloat16) # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_1: "f32[s72, 896]" = add.pow(2) mean: "f32[s72, 1]" = pow_1.mean(dim = -1, keepdim = True); pow_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_1: "f32[s72, 1]" = mean + 1e-06; mean = None rsqrt: "f32[s72, 1]" = torch.rsqrt(add_1); add_1 = None mul: "f32[s72, 896]" = add * rsqrt; add = rsqrt = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_2: "bf16[s72, 896]" = mul.to(torch.bfloat16); mul = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_1: "bf16[s72, 896]" = to_2 * _get_data_attr; to_2 = _get_data_attr = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_1: "bf16[s72, 9728]" = torch._C._nn.linear(mul_1, l_self_modules_layers_modules_23_modules_mlp_modules_gate_up_proj_parameters_weight_, None); mul_1 = l_self_modules_layers_modules_23_modules_mlp_modules_gate_up_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/activation.py:145 in forward_native, code: return F.silu(x[..., :d]) * x[..., d:] getitem: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(None, 4864, None))] silu: "bf16[s72, 4864]" = torch.nn.functional.silu(getitem); getitem = None getitem_1: "bf16[s72, 4864]" = linear_1[(Ellipsis, slice(4864, None, None))]; linear_1 = None mul_2: "bf16[s72, 4864]" = silu * getitem_1; silu = getitem_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/utils.py:105 in default_unquantized_gemm, code: return torch.nn.functional.linear(x, weight, bias) linear_2: "bf16[s72, 896]" = torch._C._nn.linear(mul_2, l_self_modules_layers_modules_23_modules_mlp_modules_down_proj_parameters_weight_, None); mul_2 = l_self_modules_layers_modules_23_modules_mlp_modules_down_proj_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:192 in forward_native, code: self.weight.data if self.has_weight else None, _get_data_attr_1: "bf16[896]" = torch._C._autograd._get_data_attr(l_self_modules_norm_parameters_weight_); l_self_modules_norm_parameters_weight_ = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:145 in forward_static, code: x = x.to(torch.float32) to_3: "f32[s72, 896]" = linear_2.to(torch.float32); linear_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:150 in forward_static, code: x = x + residual add_2: "f32[s72, 896]" = to_3 + to_1; to_3 = to_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:151 in forward_static, code: residual = x.to(orig_dtype) to_4: "bf16[s72, 896]" = add_2.to(torch.bfloat16); to_4 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:169 in forward_static, code: variance = x_var.pow(2).mean(dim=-1, keepdim=True) pow_2: "f32[s72, 896]" = add_2.pow(2) mean_1: "f32[s72, 1]" = pow_2.mean(dim = -1, keepdim = True); pow_2 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:171 in forward_static, code: x = x * torch.rsqrt(variance + variance_epsilon) add_3: "f32[s72, 1]" = mean_1 + 1e-06; mean_1 = None rsqrt_1: "f32[s72, 1]" = torch.rsqrt(add_3); add_3 = None mul_3: "f32[s72, 896]" = add_2 * rsqrt_1; add_2 = rsqrt_1 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:172 in forward_static, code: x = x.to(orig_dtype) to_5: "bf16[s72, 896]" = mul_3.to(torch.bfloat16); mul_3 = None # File: /home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/layers/layernorm.py:174 in forward_static, code: x = x * weight mul_4: "bf16[s72, 896]" = to_5 * _get_data_attr_1; to_5 = _get_data_attr_1 = None return mul_4