o 闦i)n @sUddlZddlmZmZmZddlmZddlmZm Z m Z m Z m Z m Z mZddlmZddlmZddlmZdd lmZddlZd d gZejjZd d ZiZe e e fed<ddZdOdd Zeej ddde!fddZ"eej#dPde!fddZ$eej%dPde!fddZ&eej'dPde!fddZ( dOdee!dee!dee!d e)de!f d!d"Z*eej+ej,gddde!fd#d$Z-eej.de!fd%d&Z/d'd(Z0eej1ej2ej3gddde!fd)d*Z4d+d,Z5dd-dee e e!d.fe e!d.fe e!d.fe e e!d.fffd/d0Z6dd-dee e e!d.fe e!d.fe e!d.fe e e!d.fffd1d2Z7eej8d3d4ddde!fd5d6Z9eej:d3d4de!fd7d8Z;d9d:Zej?gddde!fd;d<Z@eejAd3d4de!fd=d>ZBeejCd3d4de!fd?d@ZDiej e"ej#e$ej%e&ej'e(ej+e-ej,e-ej.e/ej1e4ej2e4ej3e4ej=e@ej>e@ej?e@ej8e9ej:e;ejAeBejCeDZdAdBZEgdCZFdDdEZGdFdGZHdHdIZIdJdKZJGdLd d ZKGdMdNdNeZLdS)QN)tree_map tree_flattentree_unflatten) ModuleTracker)ListAnyDictOptionalUnionTupleIterator) defaultdict)TorchDispatchModeprodwrapsFlopCounterModeregister_flop_formulacCst|tjr |jS|SN) isinstancetorchTensorshape)irV/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/torch/utils/flop_counter.py get_shapes r flop_registrycstddfdd }|S)N)out_valcs(tt|||f\}}}|d|i|S)N out_shape)rr)r argskwargsr!frrnfszshape_wrapper..nfrr%r&rr$r shape_wrappersr(Fcsfdd}|S)Ncs,stfdd}tjj|S)NcsHt|tjjstd|dt||tvrtd|t|<dS)Nzlregister_flop_formula(targets): expected each target to be OpOverloadPacket (i.e. torch.ops.mylib.foo), got z which is of type zduplicate registrations for )rr_opsOpOverloadPacket ValueErrortyper RuntimeError)target flop_formularrregister&s z=register_flop_formula..register_fun..register)r(rutils_pytree tree_map_)r0r1get_rawtargetsr/r register_fun"s   z+register_flop_formula..register_funr)r7r6r8rr5rr!s)r!returnc Os,|\}}|\}}||ksJ||d|S)zCount flops for matmul.r) a_shapeb_shaper!r"r#mkk2nrrrmm_flop7s rAcK t||S)zCount flops for addmm.)rA self_shaper;r<r!r#rrr addmm_flopBs rEc KsD|\}}}|\}}} ||ksJ||ksJ||| d|} | S)z"Count flops for the bmm operation.r:r) r;r<r!r#br=r>b2r?r@floprrrbmm_flopGs    rIcKrB)z&Count flops for the baddbmm operation.rIrCrrr baddbmm_flopTs rKx_shapew_shaper! transposedc CsL|d}|r|n|dd}|^}}} t|t||||d} | S)aCount flops for convolution. Note only multiplication is counted. Computation for bias are ignored. Flops for a transposed convolution are calculated as flops = (x_shape[2:] * prod(w_shape) * batch_size). Args: x_shape (list(int)): The input shape before convolution. w_shape (list(int)): The filter shape. out_shape (list(int)): The output shape after convolution. transposed (bool): is the convolution transposed Returns: int: the number of flops rr:Nr) rLrMr!rN batch_size conv_shapec_outc_in filter_sizerHrrrconv_flop_count\s   rTc Ost||||dS)zCount flops for convolution.rN)rT) rLrM_bias_stride_padding _dilationrNr!r"r#rrr conv_flopsrZc Csdd} d} | drt| d}| t|||| 7} | drIt| d}|r9| t| || || |dd7} | S| t| || || |dd7} | S)NcSs |d|dgt|ddS)Nrrr:)list)rrrrts zconv_backward_flop..trrFrU)rrT)grad_out_shaperLrMrVrWrXrYrN_output_padding_groups output_maskr!r\ flop_countgrad_input_shapegrad_weight_shaperrrconv_backward_flopsF    rdcCs|\}}}}|\}}} } |\} } } }||kr| kr8nJ||kr)| kr8nJ|| kr8| | kr8|| ks:Jd}|t||||f|||| f7}|t|||| f||| |f7}|S)z^ Count flops for self-attention. NB: We can assume that value_shape == key_shape rrJ) query_shape key_shape value_shaperFhs_qd_q_b2_h2s_k_d2_b3_h3_s3d_v total_flopsrrrsdpa_flop_counts   P""rtcOs t|||S)Count flops for self-attention.rt)rerfrgr!r"r#rrr sdpa_flops rwcCsFddlm}ddlm}t|||fs|S|g|ddS)z If the offsets tensor is fake, then we don't know the actual lengths. In that case, we can just assume the worst case; each batch has max length. r) FakeTensor)FunctionalTensorr)torch._subclasses.fake_tensorrx#torch._subclasses.functional_tensorryrdifftolistsize)offsetsmax_lenrxryrrr_offsets_to_lengths s   r)grad_out.ccs&|durt|jdksJt|jdksJ|dus#|j|jks#J|j\}} } |j\}} } |j\}} }|dus;J|dusAJ|j|jksIJt||}t||}t||D]%\}}d| || f}d| || f}d| ||f}|durt|nd}||||fVqXdS|j|j|j|dur|jndfVdS)a; Given inputs to a flash_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrlenrrzip)querykeyvaluer cum_seq_q cum_seq_kmax_qmax_k_h_qrjh_kd_kh_vrr seq_q_lengths seq_k_lengths seq_q_len seq_k_lennew_query_shape new_key_shapenew_value_shapenew_grad_out_shaperrr%_unpack_flash_attention_nested_shapess*          &rccs,|durt|jdksJt|jdksJ|dus#|j|jks#J|j\}}} } |j\}}} } |j\}}} }|dus>J|dusDJ|j|jksLJt||}t||}t||D]%\}}d| || f}d| || f}d| ||f}|durw|nd}||||fVq[dS|j|j|j|dur|jndfVdS)a? Given inputs to a efficient_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrr)rrrr cu_seqlens_q cu_seqlens_k max_seqlen_q max_seqlen_krrrjrrrrr seqlens_q seqlens_klen_qlen_krrrrrrr)_unpack_efficient_attention_nested_shapesFs*       &rT)r6c Os(t|||||||d} tdd| DS)ru)rrrrrrrcs$|] \}}}}t|||VqdSrrv.0rerfrgrrrr    z0_flash_attention_forward_flop..rsum) rrrrrrrr!r"r#sizesrrr_flash_attention_forward_flopv rc Os(t|||||||d} tdd| DS)ru)rrrrrrrcsrrrvrrrrrrz4_efficient_attention_forward_flop..rr) rrrbiasrrrrr"r#rrrr!_efficient_attention_forward_floprrcCsVd}|\}}}}|\} } } } |\} }}}|\}}}}|| kr)| kr)|krBnJ|| kr;|kr;|krBnJ|| ksDJ||krP| |krP||ksRJd}|t||||f|||| f7}|t||||f|||| f7}|t||| |f||||f7}|t|||| f||| |f7}|t||||f|||| f7}|S)NrrJ)r]rerfrgrsrFrhrirjrkrlrmrnrorprqrr_b4_h4_s4_d4rrrsdpa_backward_flop_counts    T"""""rcOst||||S)z(Count flops for self-attention backward.r)r]rerfrgr!r"r#rrrsdpa_backward_flopsrc O*t|||||||| d} tdd| DS)N)rrrrrrrrcs&|]\}}}}t||||VqdSrrrrerfrgr]rrrr   z1_flash_attention_backward_flop..r) rrrrout logsumexprrrrr"r#shapesrrr_flash_attention_backward_flop rc Or)N)rrrrrrrrcsrrrrrrrrrz5_efficient_attention_backward_flop..r) rrrrrrrrrrr"r#rrrr"_efficient_attention_backward_floprrcCst|ts|fS|Sr)rtuple)xrrrnormalize_tuple.s r)KMBTcCs0tdtttdtt|dd}t|S)Nrrr:r)maxminrsuffixesstr)numberindexrrrget_suffix_str7s(rcCs&t|}|d|d}|t|S)Niz.3f)rr)rsuffixrrrrrconvert_num_with_suffix>s  rcCs|dkrdS||dS)Nr0%z.2%r)numdenomrrrconvert_to_percent_strEs rcstfdd}|S)Ncst|\}}|}t||Sr)rr)r" flat_argsspecrr$rrr&Ks  z)_pytreeify_preserve_structure..nfrr'rr$r_pytreeify_preserve_structureJsrc seZdZdZ    ddeeejje ejjfde de dee e e fffd d Zd e fd d Zd e ee e e fffddZdddZddZddZddZZS)ra ``FlopCounterMode`` is a context manager that counts the number of flops within its context. It does this using a ``TorchDispatchMode``. It also supports hierarchical output by passing a module (or list of modules) to FlopCounterMode on construction. If you do not need hierarchical output, you do not need to use it with a module. Example usage .. code-block:: python mod = ... with FlopCounterMode(mod) as flop_counter: mod.sum().backward() Nr:Tmodsdepthdisplaycustom_mappingcstttdd|_||_||_d|_|duri}|dur&tjdddit dd| D|_ t |_ dS)NcSsttSr)rintrrrrosz*FlopCounterMode.__init__..zvrrr ys*z,FlopCounterMode.__init__..) super__init__r flop_countsrrmodewarningswarnritemsr mod_tracker)selfrrrr __class__rrrhs  zFlopCounterMode.__init__r9cCst|jdS)NGlobal)rrvaluesrrrrget_total_flops}szFlopCounterMode.get_total_flopscCsdd|jDS)aReturn the flop counts as a dictionary of dictionaries. The outer dictionary is keyed by module name, and the inner dictionary is keyed by operation name. Returns: Dict[str, Dict[Any, int]]: The flop counts as a dictionary. cSsi|] \}}|t|qSr)dictrrrrrsz3FlopCounterMode.get_flop_counts..)rrrrrrget_flop_countss zFlopCounterMode.get_flop_countsc s|durj}|dur d}ddl}d|_gd}g}tdfdd}tjD]}|dkr;q4|d d }||krGq4|||d }| |q4djvroso|D] } d | d| d<q]|dd|}t |dkrzgd g}|j||d dS)Ni?BrT)ModuleFLOPz% TotalFcstj|}|kOd|}g}|||t|t|gj|D]\}}||dt|t|t|gq,|S)N z - )rrrappendrrrr)mod_namerrspaddingrr>r global_flops global_suffixis_global_subsumedrrr process_mods  z.FlopCounterMode.get_table..process_modr.rr)r0r)leftrightr)headerscolalign) rtabulatePRESERVE_WHITESPACErrsortedrkeyscountextendr) rrr headerrrmod mod_depth cur_valuesrrrr get_tables6   zFlopCounterMode.get_tablecCs,|j|jt||_|j|Sr)rclearr __enter___FlopCounterModerrrrrrs    zFlopCounterMode.__enter__cGsD|jdusJ|jj|}d|_|j|jr t||j|Sr)r__exit__rrprintrr)rr"rFrrrrs  zFlopCounterMode.__exit__cCsV||jvr)|j|}||i|d|i}t|jjD] }|j|||7<q|S)Nr )rsetrparentsr)r func_packetrr"r#flop_count_funcraparrrr _count_flopss  zFlopCounterMode._count_flops)Nr:TNr)__name__ __module__ __qualname____doc__r r rnnrrrboolr rrrrrrrrr __classcell__rrrrrTs* = c@s$eZdZdefddZdddZdS) rcountercCs ||_dSr)r')rr'rrrrs z_FlopCounterMode.__init__rNcCs0|r|ni}|tjjjjtjjjjtjjjjtjjjjtjjjjtjjj jtjjj jtjjj jtjjj jtjjj jtjjjjtjjjjtjjjjtjjjjhvrRtS||jjvr|tjjjjur||j|i|}|turx|WdSWdn1swY||i|}|j|j|||Sr)ropsaten is_contiguousdefault memory_formatis_strides_like_formatis_non_overlapping_and_denser~sym_sizestride sym_stridestorage_offsetsym_storage_offsetnumel sym_numeldimprimlayoutNotImplementedr'rdevice decomposer_overloadpacket)rfunctypesr"r#rrrrr__torch_dispatch__s6               z#_FlopCounterMode.__torch_dispatch__)rN)r r!r"rrr@rrrrrsr)Fr)Mrtorch.utils._pytreerrrmodule_trackerrtypingrrr r r r r collectionsrtorch.utils._python_dispatchrmathr functoolsrr__all__r(r)rr__annotations__r(rmmrrAaddmmrEbmmrIbaddbmmrKr%rT convolution _convolutionrZconvolution_backwardrdrt'_scaled_dot_product_efficient_attention#_scaled_dot_product_flash_attention#_scaled_dot_product_cudnn_attentionrwrrr_flash_attention_forwardr_efficient_attention_forwardrr0_scaled_dot_product_efficient_attention_backward,_scaled_dot_product_flash_attention_backward,_scaled_dot_product_cudnn_attention_backwardr_flash_attention_backwardr_efficient_attention_backwardrrrrrrrrrrrrrs  $         ' g6 36 0