o ίiQ @sddlZddlmZddlmZddlmZmZmZddl Z ddl m Z ddl m Z ddlmZmZmZddlmZmZdd lmZGd d d eZeGd d d ZddZde jdedede jfddZde jde jdefddZde jde jdede jdee je jff ddZddZd d!Z d"d#Z! d7d$ed%e jd&ee jdefd'd(Z"Gd)d*d*e j j#Z$Gd+d,d,e j#Z%Gd-d.d.e j#Z&Gd/d0d0e j#Z'Gd1d2d2e j#Z(Gd3d4d4e j#Z)Gd5d6d6e j#Z*dS)8N) dataclass)Enum)OptionalTupleUnion)nn) functional) BlockMask_mask_mod_signatureflex_attention) AttentionBiasfmha)probec@seZdZdZdZdZdZdS) InitStdFactordisabled global_depth current_depth dim_ratioN)__name__ __module__ __qualname__DISABLED GLOBAL_DEPTH CURRENT_DEPTH DIM_RATIOrrD/home/ubuntu/.local/lib/python3.10/site-packages/core/transformer.pyrs rc@seZdZUdZeed<dZeed<dZeeed<dZ eeed<dZ eeed<dZ ee ed <d Z eed <d Ze ed <dZe ed<dZeed<dZeed<dZeed<dZeed<dZee ed<dZeed<dZeed<dS)BaseTransformerArgsidimn_layersNhead_dimn_heads n_kv_headsffn_dim_multiplier multiple_ofgh㈵>norm_eps@ rope_theta old_context_lenrope_scale_factorlow_freq_factor high_freq_factor init_base_stdrinit_std_factor max_seqlen)rrrrint__annotations__r r!rr"r#r$floatr&r'r)r+r-r.r0r1r2strr4rrrrrs"          rcKs0tjt|jddd|jddfi|S)N)end_dim)Fnll_loss log_softmaxflattenr7)predtargetkwargsrrr cross_entropy4s  rCxn_reprreturncCsj|dksJd|j\}}}}|dkr|S|dddddddddf||||||||||S)z0torch.repeat_interleave(x, dim=2, repeats=n_rep)zAOnly dim=2 is supported. Check the implementation for other dims.r,N)shapeexpandreshape)rDrErbsslenr#r!rrr repeat_kv<s rM freqs_cisseq_dimcs|jdkrksJJ|j|j|jdddfks+Jd|j|jffddt|jddDddg}|j|S) a  Reshape frequency tensor for broadcasting it with another tensor. This function reshapes the frequency tensor to have the same shape as the target tensor 'x' for the purpose of broadcasting the frequency tensor during element-wise operations. Args: freqs_cis (torch.Tensor): Frequency tensor to be reshaped. x (torch.Tensor): Target tensor for broadcasting compatibility. seq_dim (int): Sequence dimension index. Returns: torch.Tensor: Reshaped frequency tensor. rrGzfreqs_cis vs x: cs,g|]\}}|ks|dkr|ndqS)r,r).0idndimrOrr `s z)reshape_for_broadcast..Nr9)rVrH enumerateview)rNrDrOrHrrUrreshape_for_broadcastIs   rZxqxkcCs|jg|jdddddR}|jg|jdddddR}t|||}||dd}||dd}||||fS)Nr;r,rGrQ)rJrHrZr7sumr?type_as)r[r\rOrNxq_xk_xq_outxk_outrrrapply_rotary_embfs&&rdcCs||kSNr)bhq_idxkv_idxrrr causal_maskvsrjcCs |d}|d}d|d<|S)Nrr,)cumsumroll)lengths doc_startrrrlengths_to_start_idszs  rocCsV|jdksJ|d}|}t|}t|}||}tj||jd|}||fS)Nr,rdevice)rVsizer^torchrepeat_interleaveroarangerq)rmnb_seqs total_seqlendoc_idrntok_idrrrlengths_to_local_idss  rzmask_modrm kv_lengthscs\|dur|n|}t|\t|\|d|dfdd}|S)aGenerates mask mods that apply to inputs to flex attention in the sequence stacked format. Args: mask_mod: The mask mod to apply to the documents lengths: Lengths of each document Note: What is the sequence stacked format? When assembling batches of inputs, we take multiple sequences and stack them together to form 1 large sequence. We then use masking to ensure that the attention scores are only applied to tokens within the same document. Example: - Square mask doc_mask lengths a a b b b c c 2 3 2 a 1 0 0 0 0 0 0 a 1 1 0 0 0 0 0 b 0 0 1 0 0 0 0 b 0 0 1 1 0 0 0 b 0 0 1 1 1 0 0 c 0 0 0 0 0 1 0 c 0 0 0 0 0 1 1 Nr,c sbt|}t|}|k|k@}||k}|}|} |||| } || @|@Sre)rsminimum) rfrgrhri q_idx_cap kv_idx_cap valid_idxsame_doc q_logical kv_logical inner_maskkv_document_id kv_max_idx kv_token_idr{ q_document_id q_max_idx q_token_idrr doc_mask_mods   z+generate_doc_mask_mod..doc_mask_mod)rzr^)r{rmr|rrrrgenerate_doc_mask_mods     rcseZdZdZ     ddededed ed ed ed effd d ZddZddZ ddededefddZ dde ede e j fddZ ZS) RotaryEmbeddingz RotaryEmbedding Module r3r,r/r*thetar!r4 scale_factorr.r0r+csht||_||_||_||_||_||_||_|dkr0|||_ |||_ |j |j ks2JdSdS)Nr,) super__init__rr!r4rr.r0r+low_freq_wavelenhigh_freq_wavelen)selfrr!r4rr.r0r+ __class__rrrs   zRotaryEmbedding.__init__cCs&|jd|j|j|j|jddddS)NrN)rendrF) persistent)register_bufferprecompute_freqs_cisr!r4rrrrrreset_parameterss  z RotaryEmbedding.reset_parameterscCs|jdkr|Sg}|D]H}dtj|}||jkr||q ||jkr-|||jq |j|jks5J|j||j|j|j}|d|||j||q t j ||j |j dS)Nr,rG)dtyperq) rmathpirappendrr+r.r0rstensorrrq)rfreqs new_freqsfreqwavelensmoothrrr apply_scalings"     zRotaryEmbedding.apply_scalingr(rrcCsd|td|dd|d|}||}tj||jd}t||}||}}tj|| ||fddj g| ddRS)a Precompute the frequency tensor for complex exponentials (cis) with given dimensions. This function calculates a frequency tensor with complex exponentials using the given dimension 'dim' and the end index 'end'. The 'theta' parameter scales the frequencies. The returned tensor contains complex values in complex64 data type. Args: dim (int): Dimension of the frequency tensor. end (int): End index for precomputing frequencies. theta (float, optional): Scaling factor for frequency computation. Defaults to 10000.0. Returns: torch.Tensor: Precomputed frequency tensor with complex exponentials. ?rrGNrpr;r) rsrur7rrqoutercossinstackrYrr)rrrrrtrrrrrrs * 0z$RotaryEmbedding.precompute_freqs_cisNseqlentok_idxcCsH|dup|du}|sJd|dur|j|S|dur"|jd|SdS)a} Return freqs_cis corresponding to consecutive seqlen positions or the corresponding tok_idx positions Args: seqlen (int): Contiguous sequence length tok_idx (torch.Tensor[int]): Position indices of each token this overrides seqlen Returns: Tuple(torch.Tensor, torch.Tensor): Embedded input tensor and freqs_cis Nz(Should provide atleast seqlen or tok_idxr)rN)rrrtestrrrforwards  zRotaryEmbedding.forward)r3r,r,r/r*)r()NN)rrr__doc__r7r5rrrrrrsTensorr __classcell__rrrrrsL   rcsTeZdZdZddedeffdd Zdejfdd Z dejfd d Z d d Z Z S)RMSNorma Initialize the RMSNorm normalization layer. Args: dim (int): The dimension of the input tensor. eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. Attributes: eps (float): A small value added to the denominator for numerical stability. weight (nn.Parameter): Learnable scaling parameter. ư>repscs&t||_tt||_dSre)rrrr Parameterrsonesweight)rrrrrrr@s zRMSNorm.__init__rDcCs"|t||jddd|jS)Nr;T)keepdim)rsrsqrtmeanrrrDrrr_normEs"z RMSNorm._normcCs.t|d}||}||j|S)Nresid)r log_statsrr7rr_)rrDoutputrrrrHs zRMSNorm.forwardcCstjj|jdSre)rsrinitones_rrrrrrMszRMSNorm.reset_parameters)r) rrrrr5r7rrsrrrrrrrrrr2s  rcs>eZdZdejddffdd ZdejdejfddZZ S) TiedLinear tied_modulerFNcs&t||_t|dstddS)NrzPProvided module does not have attribute 'weight'. Please check your tied_module.)rrrhasattrAttributeError)rrrrrrRs  zTiedLinear.__init__rDcCst||jjSre)r<linearrrrrrr__call__ZszTiedLinear.__call__) rrrrModulerrsrrrrrrrrQsrc seZdZdededededef fdd Z   dd ejd ejd eejd ee e e e fde dejf ddZ dddZZS) Attentionrr!r"r#r)cst||_||_||_||_||_|j|j|_tj |||dd|_ tj |||dd|_ tj |||dd|_ tj |||dd|_ dS)NFbias)rrrr!r)r"r#heads_per_grouprLinearwqwkwvwo)rrr!r"r#r)rrrr_s6  zAttention.__init__NsdparDfreq_cisrmask attn_implrFcCs|j\}}}|||} |||} |||} | j} | |||j|j} | |||j|j} | |||j|j} t | | d|d|\} } t |dr[|j | | |\} } t | |jdd} t | |jdd} |dkr|duszt|tszJtdd| | | f\} } } t| | | |d } | dd} nh|d kr|dust|tsJtj| | | |d } nO|d krtd d| | | f\} } } |dust|ttjfsJt|tr|dknd}t|tjr|nd}tj| | | ||d} | dd} ntd|d|| | } | S)Nr,rkv_cacherGrr cS |ddSNr,rG transposeerrr z#Attention.forward..) block_maskr ) attn_biasrcSrrrrrrrrrcausalF) is_causal attn_maskzAttention implementation z not supported) rHrview_asrrrYr"r!r#rdrrupdaterMr isinstancer mapr r contiguousr r memory_efficient_attentionr8rsrr<scaled_dot_product_attentionNotImplementedErrorrrJ)rrDrrrrbszseq_lenrr[r\xv output_shaperrrrrrsN   zAttention.forwardrcCsn|p|jd}|j|j|jfD]}tjj|jd|d|d|dqtjj|jjd||d|d|ddSNggrPrQ)rstdarf) rrrrrr trunc_normal_rr)rinit_stdfactorwrrrrs  zAttention.reset_parametersNNrNr)rrrr5r7rrsrrrr r r8rrrrrrrr^s:, ?rc sZeZdZ ddedededeedef fdd Zd ejd ejfd d Z dddZ Z S) FeedForwardr,r hidden_dimr&r$mp_sizecsttd|d}|durt||}|||d|}||dks)J||_||_tj||dd|_tj||dd|_tj||dd|_ dS)NrGrQr,rFr) rrr5rrrrw1w3w2)rrrr&r$rrrrrs.   zFeedForward.__init__rDrFcCs8|||}|||}|t||}|Sre)rrrrr<silu)rrDx1x3rrrrrszFeedForward.forwardNrcCs|p|jd}|p |jd}|}||}|j|jfD]}tjj|jd|d|d|dqtjj|jjd|d|d|ddSr) rrrrrrrrr)rrr in_init_std out_init_stdrrrrrs& zFeedForward.reset_parameters)r,r) rrrr5rr7rrsrrrrrrrrrs#rc speZdZdeffdd Z   ddejdejdeejd eee e e fd e d ejf d d Z dddZ ZS)TransformerBlockargscst|jdus|jdusJd|jp|j|j|_|jp%|j|j|_|jp,|j|_|j|jdks8J|j|jdksBJt|j|j|j|j|jd|_t |jd|j|j |j d|_ t |j|jd|_t |j|jd|_dS)Nz+Should specify at least head_dim or n_headsr)rr!r"r#r))rrr&r$)r)rrr!r"rr#rr) attentionrr&r$ feed_forwardrr'attention_normffn_norm)rr rrrrs2   zTransformerBlock.__init__NrrDrrrrrFcCs6||j||||||d}||||}|S)Nrrr)r rr r)rrDrrrrrgoutrrrr:s zTransformerBlock.forwardrcCs4|j|||j|j|||jdSre)r rrr r)rrrrrr init_weightsMs zTransformerBlock.init_weightsrr)rrrrrrsrrrr r r8rrrrrrrr s&! r c sdeZdZdeffdd Z   ddeejdeee e e fde fd d Z d d Z d dZZS)BaseTransformerr c st|j|_|j|_t|j|_|j|_t|j|j p"|j|j |j|j |j |j |jd|_t|_t|jD] }|jt|q:dS)N)rr!r4rr.r0r+)rrrr1rr2r4rr)r!r"r-r.r0r+rope_embeddingsr ModuleListlayersranger rr )rr _rrrrVs$   zBaseTransformer.__init__NrrrrcCs:|j|j|d}t|jD] \}}||||||d}q |S)N)rrr)rr4rXr)rrgrrrrrSlayerrrrrjszBaseTransformer.forwardcCs|jdSre)rrrrrrrxsz BaseTransformer.reset_parametersc Cst|t|jD].\}}tjd|ddtjdt|jddtj|jdtj di|j }| |j |q dS)NrGr,g?ir) rrXrrrrlenrrrr2rr1)rdepthrrrrrr|s zBaseTransformer.init_weightsr)rrrrrrrsrrr r r8rrrrrrrrrUs rre)+r dataclassesrenumrtypingrrrrsrtorch.nnrr<!torch.nn.attention.flex_attentionr r r xformers.opsr r corerrrrCrr5rMrZrdrjrorzrrrrrrrr rrrrrs^        4m }A9