o }oi/8 @sddlmZddlmZddlZddlmZddlmZz"ddl m Z m Z ddl m Z mZddlmZmZddlmZd ZWneefyYdd lmZeZeZ eZeZd ZYnwdd lmZdd lmZmZddlmZddl m!Z!z ddl"m#Z$d Z%Wn eyd Z%Ynwzddl&m'Z(d Z)Gddde!dZ'Wn eyd Z)Ynwz ddl*m+Z+d Z,Wn eyd Z,YnweGdddZ-ddZ.Gdddej/Z0Gdddej/Z1Gddde1Z2Gd d!d!e1Z3Gd"d#d#ej/Z4dS)$) dataclass)UnionN) rearrange)TELayerNormColumnParallelLinearTERowParallelLinear)IdentityFuncOp IdentityOp) ModuleSpec build_module)TransformerConfigT)ApexGuardDefaultsF)activation_registry) HyenaFilterHyenaFilterSubmodules)torch_dtype_from_precision) Singleton) fftconv_func) FlashFFTConvc@seZdZddZdS)rcCst|||_dSN)FlashFFTConvImpl flashfftconv)selfseqlendtyperc/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/nlp/modules/common/hyena/hyena.py__init__NszFlashFFTConv.__init__N)__name__ __module__ __qualname__rrrrrrKs r) metaclass)causal_conv1d_fnc@s^eZdZUeZeeefed<e Z eeefed<eZ eeefed<eZ eeefed<dS)HyenaOperatorSubmodulesin_proj short_filterimplicit_filterout_projN) rrrrr#rr type__annotations__rr$r%r&rrrrr"\s r"cKs"|D] \}}t|||qdSr)itemssetattr)clskwargskvrrrauto_assign_attrsdsr/cs&eZdZdfdd ZddZZS)CausalDepthWiseConv1dTcsBtstdt||_||_tj||||d||d|_dS)NzEMissing causal-conv1d library, please run 'pip install causal-conv1d') in_channels out_channels kernel_sizepaddinggroupsbias) HAVE_CAUSAL_CONV1D ImportErrorsuperrchannelswidthnnConv1d_conv_1d)rr;r<r7 __class__rrrjs  zCausalDepthWiseConv1d.__init__cCst||jjd|jjS)Nr1)r!r?weightsqueezer7)rxrrrforwardzszCausalDepthWiseConv1d.forward)T)rrrrrE __classcell__rrr@rr0isr0c sFeZdZdedfdededededeeefde f fd d Z Z S) HyenaConvTNd_modelmax_seq_lengthorderr7 filter_clsfilter_submodulesc  st||_||_||_||_|j|jd}|jr&tt ||_ nt ||_ t ||j|jdf||d||_ dS)Nr1) submodulesseq_len)r:rrHrJrIuse_biasr= Parametertorchrandnr7zerosr filter) rrHrIrJr7rKrL filter_kwargs bias_shaper@rrrs$   zHyenaConv.__init__) rrrrintboolrr r'rrrFrrr@rrG~s" rGcsjeZdZdedddfdededededeeefd e d e d e ffd d Z ddZ ddZ ddZZS)SingleHeadHyenaConvTNbf16rHrIrJr7rKrL fftconv_type precisionc  stj|||f|||d| |dur|dkrtrd}nd}|dvr'td|dkr3|dkr3td|dkr=ts=td|dkrGtsGtd |dkrQ|j|_dStd |j t |j |_ |j |_dS) Nr7rKrLi safariflash)r^r_z/fftconv_type must be one of ['safari', 'flash']z7Safari-fftconv only supports sequence length up to 8192NSafari-fftconv library not found. Please see README at for instructions.zLflashfftconv library not found. Please see README at for instructions.) r:rHAVE_SAFARI_FFTCONV ValueErrorr9HAVE_FLASHFFTCONV _safari_fft fftconv_fnrrIrr _flash_fft) rrHrIrJr7rKrLr[r\rUr@rrrs>      zSingleHeadHyenaConv.__init__cCs|jtjd}t|||ddS)NrF)gelu)torQfloat32safari_fftconv_fn)rrDr-r7rrrreszSingleHeadHyenaConv._safari_fftcCs(|}|||||jdd}|SNr1dim) contiguousr unsqueeze)rrDr-r7yrrrrgszSingleHeadHyenaConv._flash_fftcCs0t|jd|j|jdd|}||||}|S)Nz (v o) -> o vr1r.o)rr7rHrJrf)rrDr-recurrence_idxr7rrrrrrEszSingleHeadHyenaConv.forward)rrrrrWrXrr r'rstrrrergrErFrrr@rrYs4  .rYcs^eZdZdedddfdededededed eeefd e d e d e ffd d Z ddZ Z S)MultiHeadHyenaConvTNrZrHrIrJ num_headsr7rKrLr[r\c  s`|dkrtd|dkrtd|jdtstdtj|||f|||d| ||_dS)Nr1zExpecting num_heads > 1raz5Multi-head supported only with order == 2 (got order )r`r])rcrJrbr9r:rrx) rrHrIrJrxr7rKrLr[r\rUr@rrrs$  zMultiHeadHyenaConv.__init__c Cs.|jjtjd}t|||dd||j|d}|S)NrhFT)rioutput_hbl_layoutr.head_dimq)r7rjrQrkrlrx)rr.r-x1x2r7rrrrrrEszMultiHeadHyenaConv.forward)rrrrrWrXrr r'rrvrrErFrrr@rrws4   rwcsfeZdZ       ddeded ed ed ed ed edeffdd ZddZ ddZ ddZ Z S) HyenaOperatorrar1identityNconfigrIrJrxdropoutshort_filter_order activationrMc  st|durtttttd}|dkrtd|jd|j } | |dkr1td| d|| |} t || |||| |||d t ||_ t ||_t|j|j|jd |j|j|jjd d d d d d |_t|j|j|j|j|jjd d d d dd |_|j|jd } t|j| |j|_|j|j|jg}|j| d<|jj| d<|jd krt|i| |_|j |_!dS|"|jt#|i| |_|j$|_!dS)aF Hyena operator described in the paper https://arxiv.org/pdf/2302.10866.pdf Args: max_seq_length: (int): Maximum input sequence length. order: (int): Depth of the Hyena recurrence. Defaults to 2 num_heads: (int): Number of heads. Defaults to 1 dropout: (float): Dropout probability. Defaults to 0.0 short_filter_order: (int): Length of the explicit input convolutional filter. Defaults to 3 activation: (str): type of act between kernel output and output projection (default identity) N)r#r$r%r&razOrder must be at least 2, (got ryrzModel dimension z must be divisible by num heads )rHrJrIrxr{rr mcore_configr1FTr#)r init_method gather_outputr7 skip_bias_add is_experttp_comm_buffer_namer&)rrr7input_is_parallelrrrrKrL)%r:rr"rr0rrrcrJ hidden_sizer/r rr=Dropoutrr r#rHrrr&output_layer_init_methodr$rr{rIr%rMrxrY long_convconv_single_head conv_fwd_fnappendrwconv_multi_head)rrrIrJrxrrrrM layer_numberlong_conv_kwargsrHr{ total_widthlong_conv_argsr@rrrs|          zHyenaOperator.__init__c Os|d}t||j}||}t|tr|dn|}t|d}|j|}t|d|j dd}| |dd|f}|j t j d}|||}t|d}||}||}t|tra|\}} nd} t|d}|| fS) Nrzl b d -> b d lzc l v -> c v l)r..rhzb d l -> b l dzb l d -> l b d)sizeminrIr# isinstancetuplerrrTr{r$rjrQrkrrr&) ruargsr,ll_filterr-ucrrr7rrrrEes$            zHyenaOperator.forwardcCs~t|d|j|jdd}|j|jdd^}}tt|ddD]\}}|||}|j||||d}q!||d}|S)Nz(o v) l -> o v lr1rsrn)r-rur) rr{rJsplitrH enumeratereversedrr)rrr-rDr.rtx_irrrrrrs zHyenaOperator.conv_single_headcCsB|j|jdd\}}}|}|}|}|||||}|Srm)rrHrpr)rrr-r}r~r.rrrrrrs zHyenaOperator.conv_multi_head)rar1rrrNN) rrrr rWfloatrvr"rrErrrFrrr@rrs8 c r)5 dataclassesrtypingrrQtorch.nnr=einopsr+megatron.core.extensions.transformer_enginerr%megatron.core.transformer.identity_oprr$megatron.core.transformer.spec_utilsr r ,megatron.core.transformer.transformer_configr HAVE_MEGATRON_COREr9ModuleNotFoundError2nemo.collections.nlp.modules.common.megatron.utilsr #nemo.collections.common.parts.utilsr 6nemo.collections.nlp.modules.common.hyena.hyena_filterrr&nemo.collections.nlp.parts.utils_funcsrnemo.utils.metaclassesr9nemo.collections.nlp.modules.common.hyena.fftconv_wrapperrrlrbrrrrd causal_conv1dr!r8r"r/Moduler0rGrYrwrrrrrsd !              >&