o Xi|+@sddlmZddlmZmZddlZddlmZm Z m Z ee ej fZ Gddde jZddd DZe d deDZeeZdS) ) annotations)SequenceUnionN) _fusion_utils _ir_utilspatterncsZeZdZd fdd ZddZ             dd d Z   dd d ZZS)AttentionFusionhas_pastboolno_slicecst|||_||_dSN)super__init__ _has_past _no_slice)selfnamer r  __class__]/home/ubuntu/.local/lib/python3.10/site-packages/onnxscript/rewriter/ort_fusions/attention.pyrs  zAttentionFusion.__init__c Csx|jr||| }|||}|||}n-|j||dgd}|j|||dgdgd}|j|| | dgdgd}|j|| | dgdgd}|jr|j|dgdgdgd gd}||dg}|j|dgdgdgd gd}||dg}|j||||dtjd d d |||dgdd \}}}||dg}||dg}|j ||dd}||fS|j||||dtjd d d dd|ddgd }|S)N projected)_outputsquery_mm_sliced key_mm_slicedvalue_mm_slicedrpast_key_slicedpast_value_slicedattention_biasT)can_match_none com.microsoft) mha_output present_key present_value) num_heads_domainraxisr#) rMatMulSlicerSqueezeMultiHeadAttentionrVar UnsqueezeConcat)ropinput qkv_weightqkv_biaspastr&start1end1start2end2start3end3q_mulk_mulv_mul query_BSDkey_BSD value_BSDrpast_key past_value attentionr$r%presentrrrrs      zAttentionFusion.patternNc st}i_d(fdd }||gd r|d |d |Sjs|dus2|jdus2t|jd kr8|d |S|jdttsH|d|St |drlt | t | krlt | t | krlt | fddsr|d|S||ddgr|d |d|S||gdr|d |d|S||gdr|d |d|S||gdr|d |d|Sn3||ddgr|d |d|S||ddgr|d |d |S||dd!gr|d |d"|Sj d}j d}j d}j d!}t|trt|trt|ts|d#SjsBt|ts(|d$S||||krB|d%|d&|d'|d'|S|S))Nvalir.Valuedims Sequence[str]returnr cstj|| Sr )rcheck_shape_boolbindings)rFrH)rrrno_matchsz'AttentionFusion.check..no_match)BSDzShape mismatch: z3 does not match expected dimensions ['B', 'S', 'D']z$Input projection is not a 3D tensor.rzHidden size is not an integer.rcs|kSr r)x) hidden_sizerrsz'AttentionFusion.check..zPProjected input is not being split into q, k, v correctly based on hidden sizes.rPDhz/ does not match expected dimensions ['D', 'Dh'])rNrODh_qz6 does not match expected dimensions ['B', 'S', 'Dh_q'])rNrODh_kz6 does not match expected dimensions ['B', 'S', 'Dh_k'])rNrODh_vz6 does not match expected dimensions ['B', 'S', 'Dh_v']rVz1 does not match expected dimensions ['D', 'Dh_q']rWz1 does not match expected dimensions ['D', 'Dh_k']rXz1 does not match expected dimensions ['D', 'Dh_v']z>Could not determine the hidden sizes of query, key, and value.z4Could not determine the total hidden size of weight.zBHidden size of query, key and value do not add up to hidden size: z != z + )rFrGrHrIrJr ) r MatchResultrLfailrshapelen isinstanceintris_singleton_valueget_singleton_valueget)rr1r2r3rrrrr6r7r8r9r:r;r<r=r>_ check_resultrMrUrVrWrXr)rSrrchecks                  $ zAttentionFusion.checkc Ks|jd} |jd}|jd}| ||g}|jr#|j| | | dd}|jdd}|jrE|j|||d|||||ddd \}}||fS|j|||dd|d|||ddd S) NrVrWrXrr(scaler"r)r&qkv_hidden_sizesrer'r) rLrarr0producer attributes get_floatr Attention)rr1r2r3r4r5r r&r#r<r=r>rbrVrWrXrfrerDrErrrrewritesH     zAttentionFusion.rewrite)r r r r ) NNNNNNNNNNNNN)NNN)__name__ __module__ __qualname__rrrdrk __classcell__rrrrrs* s }rcCsBg|]}dD]}d|r dnd|rdndd||dqqS)FT attention_ with_past_r rb)rr r )strip).0r r rrr ;s rvrpcCs(g|]}tj|d|d|ddqS)rr r )r r )rrule)ruparamsrrrrvIs) __future__rtypingrronnx_irironnxscript.rewriterrrrr^ SymbolicDimDimRewriteRuleClassBaserparameter_combinationsRewriteRuleSetattention_rulesapply_fusion_rulesfuse_attentionrrrrs" ,