o ,ib@sUddlZddlmZmZmZddlmZddlmZm Z m Z m Z m Z m Z mZddlmZddlmZddlmZdd lmZdd lmZddlZd d gZejjZd dZiZe e e fed<ddZ dLdd Z!e!ej"ddde#fddZ$e!ej%dMde#fddZ&e!ej'dMde#fddZ(e!ej)dMde#fddZ* dLdee#dee#d ee#d!e+de#f d"d#Z,e!ej-ej.gddde#fd$d%Z/e!ej0de#fd&d'Z1d(d)Z2e!ej3ej4gddde#fd*d+Z5dd,dee e e#d-fe e#d-fe e#d-fe e e#d-fffd.d/Z6dd,dee e e#d-fe e#d-fe e#d-fe e e#d-fffd0d1Z7e!ej8d2d3ddde#fd4d5Z9e!ej:d2d3de#fd6d7Z;d8d9Zgddde#fd:d;Z?e!ej@d2d3de#fdd?ZCej"e$ej%e&ej'e(ej)e*ej-e/ej.e/ej0e1ej3e5ej4e5ej=e?ej>e?ej8e9ej:e;ej@eAejBeCiZd@dAZDgdBZEdCdDZFdEdFZGdGdHZHdIdJZIGdKd d eZJdS)NN)tree_map tree_flattentree_unflatten) ModuleTracker)ListAnyDictOptionalUnionTupleIterator) defaultdict)TorchDispatchMode)register_decompositionprodwrapsFlopCounterModeregister_flop_formulacCst|tjr |jS|SN) isinstancetorchTensorshape)irV/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/utils/flop_counter.py get_shapes r flop_registrycstddfdd }|S)N)out_valcs(tt|||f\}}}|d|i|S)N out_shape)rr)r!argskwargsr"frrnfszshape_wrapper..nfrr&r'rr%r shape_wrappersr)Fcsfdd}|S)Ncs"st|}ttdd||S)NT)registryunsafe)r)rr ) flop_formulaget_rawtargetsrr register_fun"sz+register_flop_formula..register_funr)r/r.r0rr-rr!s)r"returnc Os,|\}}|\}}||ksJ||d|S)zCount flops for matmul.r) a_shapeb_shaper"r#r$mkk2nrrrmm_flop*s r9cK t||S)zCount flops for addmm.)r9 self_shaper3r4r"r$rrr addmm_flop5s r=c KsD|\}}}|\}}} ||ksJ||ksJ||| d|} | S)z"Count flops for the bmm operation.r2r) r3r4r"r$br5r6b2r7r8floprrrbmm_flop:s    rAcKr:)z&Count flops for the baddbmm operation.rAr;rrr baddbmm_flopGs rCx_shapew_shaper" transposedc CsL|d}|r|n|dd}|^}}} t|t||||d} | S)aCount flops for convolution. Note only multiplication is counted. Computation for bias are ignored. Flops for a transposed convolution are calculated as flops = (x_shape[2:] * prod(w_shape) * batch_size). Args: x_shape (list(int)): The input shape before convolution. w_shape (list(int)): The filter shape. out_shape (list(int)): The output shape after convolution. transposed (bool): is the convolution transposed Returns: int: the number of flops rr2Nr) rDrEr"rF batch_size conv_shapec_outc_in filter_sizer@rrrconv_flop_countOs   rLc Ost||||dS)zCount flops for convolution.rF)rL) rDrE_bias_stride_padding _dilationrFr"r#r$rrr conv_flopvsrRc Csdd} d} | drt| d}| t|||| 7} | drIt| d}|r9| t| || || |dd7} | S| t| || || |dd7} | S)NcSs |d|dgt|ddS)Nrrr2)list)rrrrts zconv_backward_flop..trrFrM)rrL)grad_out_shaperDrErNrOrPrQrF_output_padding_groups output_maskr"rT flop_countgrad_input_shapegrad_weight_shaperrrconv_backward_flop|sF    r\cCs|\}}}}|\}}} } |\} } } }||kr| kr8nJ||kr)| kr8nJ|| kr8| | kr8|| ks:Jd}|t||||f|||| f7}|t|||| f||| |f7}|S)z^ Count flops for self-attention. NB: We can assume that value_shape == key_shape rrB) query_shape key_shape value_shaper>hs_qd_q_b2_h2s_k_d2_b3_h3_s3d_v total_flopsrrrsdpa_flop_counts   P""rlcOs t|||S)Count flops for self-attention.rl)r]r^r_r"r#r$rrr sdpa_flops ro)grad_out.ccsJ|durt|jdksJt|jdksJ|dus#|j|jks#J|j\}} } |j\}} } |j\}} }|dus;J|dusAJ|j|jksIJ|dd|dd}|dd|dd}t||D]%\}}d| || f}d| || f}d| ||f}|dur|nd}||||fVqjdS|j|j|j|dur|jndfVdS)a; Given inputs to a flash_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrlenrtolistzip)querykeyvaluerp cum_seq_q cum_seq_kmax_qmax_k_h_qrbh_kd_kh_vrj seq_q_lengths seq_k_lengths seq_q_len seq_k_lennew_query_shape new_key_shapenew_value_shapenew_grad_out_shaperrr%_unpack_flash_attention_nested_shapess*        &rccsP|durt|jdksJt|jdksJ|dus#|j|jks#J|j\}}} } |j\}}} } |j\}}} }|dus>J|dusDJ|j|jksLJ|dd|dd}|dd|dd}t||D]%\}}d| || f}d| || f}d| ||f}|dur|nd}||||fVqmdS|j|j|j|dur|jndfVdS)a? Given inputs to a efficient_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrrrrs)rwrxryrp cu_seqlens_q cu_seqlens_k max_seqlen_q max_seqlen_kr~rrbrrrrj seqlens_q seqlens_klen_qlen_krrrrrrr)_unpack_efficient_attention_nested_shapes+s*     &rT)r.c Os(t|||||||d} tdd| DS)rm)rwrxryrzr{r|r}cs$|] \}}}}t|||VqdSrrn.0r]r^r_r~rrr u   z0_flash_attention_forward_flop..rsum) rwrxryrzr{r|r}r"r#r$sizesrrr_flash_attention_forward_flop[ rc Os(t|||||||d} tdd| DS)rm)rwrxryrrrrcsrrrnrrrrrrz4_efficient_attention_forward_flop..rr) rwrxrybiasrrrrr#r$rrrr!_efficient_attention_forward_flop{rrcCsVd}|\}}}}|\} } } } |\} }}}|\}}}}|| kr)| kr)|krBnJ|| kr;|kr;|krBnJ|| ksDJ||krP| |krP||ksRJd}|t||||f|||| f7}|t||||f|||| f7}|t||| |f||||f7}|t|||| f||| |f7}|t||||f|||| f7}|S)NrrB)rUr]r^r_rkr>r`rarbrcrdrerfrgrhrirj_b4_h4_s4_d4rrrsdpa_backward_flop_counts    T"""""rcOst||||S)z(Count flops for self-attention backward.r)rUr]r^r_r"r#r$rrrsdpa_backward_flopsrc O*t|||||||| d} tdd| DS)N)rwrxryrprzr{r|r}cs&|]\}}}}t||||VqdSrrrr]r^r_rUrrrr   z1_flash_attention_backward_flop..r) rprwrxryout logsumexprzr{r|r}r#r$shapesrrr_flash_attention_backward_flop rc Or)N)rwrxryrprrrrcsrrrrrrrrrz5_efficient_attention_backward_flop..r) rprwrxryrrrrrrr#r$rrrr"_efficient_attention_backward_floprrcCst|ts|fS|Sr)rtuple)xrrrnormalize_tuples r)KMBTcCs0tdtttdtt|dd}t|S)Nrrr2rq)maxminrtsuffixesstr)numberindexrrrget_suffix_strs(rcCs&t|}|d|d}|t|S)Niz.3f)rr)rsuffixrryrrrconvert_num_with_suffixs  rcCs|dkrdS||dS)Nr0%z.2%r)numdenomrrrconvert_to_percent_str&s rcstfdd}|S)Ncst|\}}|}t||Sr)rr)r# flat_argsspecrr%rrr',s  z)_pytreeify_preserve_structure..nfrr(rr%r_pytreeify_preserve_structure+src seZdZdZ    ddeeejje ejjfde de dee e e ffd d Zd e fd d Zd e ee e e fffddZdddZfddZfddZdddZddZZS)ra ``FlopCounterMode`` is a context manager that counts the number of flops within its context. It does this using a ``TorchDispatchMode``. It also supports hierarchical output by passing a module (or list of modules) to FlopCounterMode on construction. If you do not need hierarchical output, you do not need to use it with a module. Example usage .. code-block:: python mod = ... with FlopCounterMode(mod) as flop_counter: mod.sum().backward() Nr2Tmodsdepthdisplaycustom_mappingcCsdtdd|_||_||_|duri}|durtjddditdd|D|_t|_ dS)NcSsttSr)rintrrrrOsz*FlopCounterMode.__init__..zXs*z,FlopCounterMode.__init__..) r flop_countsrrwarningswarnr itemsr mod_tracker)selfrrrrrrr__init__Is zFlopCounterMode.__init__r1cCst|jdS)NGlobal)rrvaluesrrrrget_total_flops\szFlopCounterMode.get_total_flopscCsdd|jDS)aReturn the flop counts as a dictionary of dictionaries. The outer dictionary is keyed by module name, and the inner dictionary is keyed by operation name. Returns: Dict[str, Dict[Any, int]]: The flop counts as a dictionary. cSsi|] \}}|t|qSr)dictrrrrrisz3FlopCounterMode.get_flop_counts..)rrrrrrget_flop_counts_s zFlopCounterMode.get_flop_countsc s|durj}|dur d}ddl}d|_gd}g}tdfdd}tjD]}|dkr;q4|d d }||krGq4|||d }| |q4djvrwswt |D]\} } d || d|| d<q_|dd|}t |dkrgd g}|j||d dS)Ni?BrT)ModuleFLOPz% TotalFcstj|}|kOd|}g}|||t|t|gj|D]\}}||dt|t|t|gq,|S)N z - )rrrappendrrrr)mod_namerrkpaddingrr6r global_flops global_suffixis_global_subsumedrrr process_modys  z.FlopCounterMode.get_table..process_modr.rr)r0r)leftrightr)headerscolalign) rtabulatePRESERVE_WHITESPACErrsortedrkeyscountextend enumeratert) rrrheaderrrmod mod_depth cur_valuesidxryrrr get_tableks6   zFlopCounterMode.get_tablecs"|j|jt|Sr)rclearr __enter__superr __class__rrrs   zFlopCounterMode.__enter__cs4tj||j|jrt||jdSdSr)r__exit__rrprintrr)rr#rrrrs  zFlopCounterMode.__exit__rcCs,|r|ni}||i|}||j|||Sr) _count_flops_overloadpacket)rfunctypesr#r$rrrr__torch_dispatch__s z"FlopCounterMode.__torch_dispatch__cCsV||jvr)|j|}||i|d|i}t|jjD] }|j|||7<q|S)Nr!)r setrparentsr)r func_packetrr#r$flop_count_funcrYparrrrr s  zFlopCounterMode._count_flops)Nr2TNr)rN)__name__ __module__ __qualname____doc__r r rnnrrrboolr rrrrrrrrrr  __classcell__rrrrr5s,  <  )Fr)Krtorch.utils._pytreerrrmodule_trackerrtypingrrr r r r r collectionsrtorch.utils._python_dispatchr torch._decomprmathr functoolsrr__all__opsatenrr __annotations__r)rmmrr9addmmr=bmmrAbaddbmmrCrrL convolution _convolutionrRconvolution_backwardr\rl'_scaled_dot_product_efficient_attention#_scaled_dot_product_flash_attentionrorr_flash_attention_forwardr_efficient_attention_forwardrr0_scaled_dot_product_efficient_attention_backward,_scaled_dot_product_flash_attention_backwardr_flash_attention_backwardr_efficient_attention_backwardrrrrrrrrrrrrs  $           ' g 6 36 0      !