o i@#@s:UddlZddlmZmZmZddlmZddlZddlm Z edZ edZ iZ e ejjefed<ehdZd ed ed eee e fgee e fffd d Zedd dN)CallableOptionalTypeVar) ParamSpec)_dtype_mappings_P_RONNX_ATEN_DECOMP_TABLE> op_type opset_versionreturncs,dtttfdtttfffdd }|S)zDDecorator to register an ONNX operator with a custom implementation.funcrcsPd}tjjdd|dd|}|ttttjj|<|||S)Nopsetzonnx::.) mutates_args)torchlibrary custom_opr getattropsonnx register_fake)roverloadtorch_oprrrQ/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/torch/onnx/ops/_impl.py decorators  z_onnx_op..decorator)rrr)rrr!rrr _onnx_ops( r"RotaryEmbeddingF) interleaved num_headsrotary_embedding_dimx cos_cache sin_cache position_idsr%r&r'cs|jt}dddurWtdkfddtjdkfddtjdkfd dtdkoNdkfd dntd kodd kfd d|d krwt|d}n$|d krt|dkfddd}||} || g} t|| }tt|jd kdd|jd } |dkr| }|ddddddd|f} |dddddd|df} |ddurntjdkojdkfddtjdkojdkfddtjdkfddtjdkfddtdtd|rk| dddddddddf} | dddddddddf}n tj| ddd\} }| |}| |}|rt|d}t|d}tj ||fdd}t|| j} n tj ||fdd} tj | | fdd}|d krt|St|dS)z_RotaryEmbedding-23 https://onnx.ai/onnx/operators/onnx__RotaryEmbedding.html#rotaryembedding-23rNcs djS)Nz6position_ids must be 2D when provided. Received shape shaper)r+rr Fs z%rotary_embedding_23..csddjdS)Nz6position_ids first dim (batch) must match x.shape[0] ( ). Received rr.r) batch_sizer+rr r0Jr csddjdS)Nz;position_ids second dim (sequence) must match x.shape[-2] (r1r r.r)r+sequence_lengthrr r0Nr3cdjdjS)NzWcos_cache/sin_cache must be 2D when position_ids is provided. Received cos_cache shape , sin_cache shape r.rr)r*rr r0R cr5)Nz[cos_cache/sin_cache must be 3D when position_ids is not provided. Received cos_cache shape r6r.rr7rr r0Xr8)rr-r r9cs dS)NzKnum_heads must be provided for 3D inputs. Received input tensor with shape rr) input_shaperr r0es cSdS)Nzx should be a 4D tensor by nowrrrrr r0lcsdjdddS)Nzcos has shape  but expected (batch=, seq=, ...)r.r)r2cosr4rr r0csdjdddS)Nzsin has shape r>r?r@r.r)r2r4sinrr r0rBcsdjdddS)NzLast dimension of cos cache (rD') should match rotary_embedding_dim/2 ().r.r)rArotary_embedding_dim_halfrr r0csdjdddS)NzLast dimension of sin cache (rDrErFr.r)rGrCrr r0rHdim) r/lenr_checkrJpermutereshape unsqueezechunkcat)r(r)r*r+r%r&r' input_rank hidden_size head_size new_shapex_rotate x_not_rotatex1x2realimagx_rotate_concatoutputr) r2rAr)r;r+rGr4rCr*r rotary_embedding_23/s                  "$     r^scalerTcCs|dur|Sdt|S)z/Get the scale factor for attention computation.Ng?)mathsqrt)r_rTrrr _get_scale_factorsrbtensorr2cCs:|jd|jd}}||}|||||ddS)z1Reshape 3D tensor to 4D for multi-head attention.r r-)r/view transpose contiguous)rcr2r&r4rSrTrrr _reshape_3d_to_4ds rgQKcurrent_q_num_headscurrent_kv_num_headsqk_matmul_output_modec Cs2|dkr t|||||Stt||ddS)z1Get QK output tensor based on the specified mode.rr,rD)_compute_qk_output_for_mode_0r zeros_likematmulre)rhrirjrkr_rlrrr _get_qk_output_for_aten_spdas   rpcs"tdkfdddS)z-Validate Group Query Attention configuration.rcsdddS)Nz q_num_heads (z%) must be divisible by kv_num_heads (z ) for GQArrrkrjrr r0sz-_validate_gqa_configuration..N)rrL)rjrkrrqr _validate_gqa_configurations  rrc Cs`|}||kr||}|j|dd}t||jd}t|}||} ||} t| | ddS)zDHelper function to compute QK output for qk_matmul_output_mode == 0.r rIr9r,rD)repeat_interleaverbr/r`rarrore) rhrirjrkr_K_for_qk repeat_factor scale_factor sqrt_scaleQ_scaledK_scaledrrr rms  rm Attention) is_causal kv_num_heads q_num_headsrlr_softcapsoftmax_precisionV attn_maskpast_key past_valuer|r}r~rrc( Csd\} }}t|j}|jd}t|jdkr;t|dko|dkdd|jd}t|||}t|||}t|||}tt|jdkoQt|jdkoQt|jdkdd|j|}t| |} |d urmtj||g|d n|}|d ur~tj||g|d n|}||}}|j| }|j| }|j|}|j|}| d ko| dko| d uo|d up|jtj k}t |||rd }|d ur|jtj kr|n|}tj j j ||||d || t ||kd }t||||| | }n||kr||}|j|| d }|j|| d }tj|||j|jd }|r+t|d uddttj||tj |jd }||td}|d urE|jtj krA||td}n||}t| |jd} t| }!||!}"||!}#t|"|#dd}$|$}|$|}%| dkrq|%}| d kr| t|%| }%| dkr|%}| d ur| tvr|%j}&|%tj| }%tj|%dd }'|'|&}'ntj|%dd }'ntj|%dd }'| dkr|'}t|'|}|dkr|dd !||d}||||fS)zMAttention-23 https://onnx.ai/onnx/operators/onnx__Attention.html#attention-23)r r-r9rr9cSr<)Nz;q_num_heads and kv_num_heads must be provided for 3D inputsrrrrr r0r=zattention_23..r r:cSr<)Nz'Q, K, and V should be 4D tensors by nowrrrrr r0!r=NrIr{)r dropout_pr|r_ enable_gqa)dtypedevicecSr<)Nz'Cannot use both is_causal and attn_maskrrrrr r0vr=z-infr,rDr-)"rKr/rrLrgrbrQclonerboolrrnn functionalscaled_dot_product_attentionrprszerosrtrilones masked_fillfloatr`raroretanh-_ATTENTION_23_ALLOWED_INTERMEDIATE_PRECISIONStorONNX_DTYPE_TO_TORCH_DTYPEsoftmaxrfrd)(rhrirrrrr|r}r~rlr_rr num_head_dim sequence_dimhead_diminput_shape_lenr2q_sequence_length q_head_size present_key present_valuerjrkkv_sequence_length can_use_sdpasdpa_attn_maskr] qk_outputru attn_bias causal_maskrvrwrxryqk_matmul_output qk_with_biasoriginal_dtype qk_softmaxrrr attention_23s       (                        r)N)NNN)!r`typingrrrtyping_extensionsrrtorch.onnx.opsrrrr dict_ops OpOverload__annotations__ frozensetrstrintr"Tensorrr^rrbrgrprrrmtuplerrrrr s