o piO@sddlZddlZddlmZddlZddlmZdde fddZ ej j j ddfde de fdd ZGd d d ejZGd d d ejZGdddejZGdddejZGdddejZdS)Nkey_chunk_sizecsjdd\}jdt|ttjtjddfddfdd }tjj |t d |d \}}} tj | d d d } t | | } |tj | dd9}|| 9}|jd d} t |djd d} | | S)zBMulti-head dot product attention with a limited number of queries.NF) prevent_csecsltjd||d}tj|ddd}tj|}t||}tjd||d}td|}||jdd|fS) Nz...qhd,...khd->...qhk precisionrTaxiskeepdimsz...vhf,...qhv->...qhfz ...qhk->...qhr )jnpeinsummaxjaxlax stop_gradientexpsum)querykeyvalue attn_weights max_score exp_weights exp_valuesr]/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/diffusers/models/attention_flax.pysummarize_chunks  z/_query_chunk_attention..summarize_chunkcstjjdgjd|ddgtjddgd}tjjdgjd|ddgtjddgd}||S)Nrroperand start_indices slice_sizes)rr dynamic_slicendimlistshape) chunk_idx key_chunk value_chunk) k_featuresrr num_headsrr v_featuresrrr chunk_scanner+s z-_query_chunk_attention..chunk_scannerr)fxsTr r )r'minr sqrt functoolspartialr checkpointrmaparangerr expand_dimsr)rrrrrnum_kvr. chunk_values chunk_weights chunk_max global_max max_diffs all_values all_weightsr) r+rrr,rrrr-rr_query_chunk_attentions    rAiquery_chunk_sizec s`jdd\f dd}tjj|ddtd\}}tj|ddS)a Flax Memory-efficient multi-head dot product attention. https://arxiv.org/abs/2112.05682v2 https://github.com/AminRezaei0x443/memory-efficient-attention Args: query (`jnp.ndarray`): (batch..., query_length, head, query_key_depth_per_head) key (`jnp.ndarray`): (batch..., key_value_length, head, query_key_depth_per_head) value (`jnp.ndarray`): (batch..., key_value_length, head, value_depth_per_head) precision (`jax.lax.Precision`, *optional*, defaults to `jax.lax.Precision.HIGHEST`): numerical precision for computation query_chunk_size (`int`, *optional*, defaults to 1024): chunk size to divide query array value must divide query_length equally without remainder key_chunk_size (`int`, *optional*, defaults to 4096): chunk size to divide key and value array value must divide key_value_length equally without remainder Returns: (`jnp.ndarray`) with shape of (batch..., query_length, head, value_depth_per_head) rNcs`tjjdgjd|ddgtjddtgd}|t|dfS)Nrrrr )rrrrr)rrr$r%r&r'r1rA)r(_ query_chunk rrr,num_qr q_featuresrrBrrrr.as  z5jax_memory_efficient_attention..chunk_scannerr)r/initr0lengthr )r'rrscanmathceilr concatenate) rrrrrBrr.rCresrrErjax_memory_efficient_attentionJs  rOc@seZdZUdZeed<dZeed<dZeed<dZe ed<d Z e ed <d Z e ed <e jZe jed <d dZddZddZdddZdS) FlaxAttentiona A Flax multi-head attention module as described in: https://arxiv.org/abs/1706.03762 Parameters: query_dim (:obj:`int`): Input hidden states dimension heads (:obj:`int`, *optional*, defaults to 8): Number of heads dim_head (:obj:`int`, *optional*, defaults to 64): Hidden states dimension inside each head dropout (:obj:`float`, *optional*, defaults to 0.0): Dropout rate use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): enable memory efficient attention https://arxiv.org/abs/2112.05682 split_head_dim (`bool`, *optional*, defaults to `False`): Whether to split the head dimension into a new axis for the self-attention computation. In most cases, enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): Parameters `dtype` query_dimheads@dim_headdropoutFuse_memory_efficient_attentionsplit_head_dimdtypecCs|j|j}|jd|_tj|d|jdd|_tj|d|jdd|_tj|d|jdd|_tj|j |jdd|_ tj |j d |_ dS) NgFto_q)use_biasrZnameto_kto_vto_out_0)rZr]rate)rUrSscalennDenserZrrrrQ proj_attnDropoutrW dropout_layerself inner_dimrrrsetups  zFlaxAttention.setupcCsL|j\}}}|j}||||||}t|d}||||||}|SN)rrr'rSreshaper transposerjtensor batch_sizeseq_lendim head_sizerrrreshape_heads_to_batch_dim  z(FlaxAttention.reshape_heads_to_batch_dimcCsL|j\}}}|j}||||||}t|d}||||||}|Srmrprsrrrreshape_batch_dim_to_headsrzz(FlaxAttention.reshape_batch_dim_to_headsNTcCs|dur|n|}||}||}||}|jrD|jd}t||d|j|jf}t||d|j|jf} t||d|j|jf} n| |}| |} | |} |j r| ddd}| ddd} | ddd} |jd} | ddkr}t | d} n| ddkrt | d} n| ddkrt | d} nt | } t || | | d d }| ddd}nL|jrtd | |} ntd || } | |j} tj| |jrdndd }|jrtd|| }|jd}t||d|j|jf}n td|| }||}||}|j||dS)NrrrornrrTi@)rBrzb t n h, b f n h -> b n f tzb i d, b j d->b i jr zb n f t, b t n h -> b f n hzb i j, b j d -> b i d deterministic)rrrrYr'r rqrSrUryrXrrintrOrrcrdsoftmaxr{rfrh)rj hidden_statescontextr query_projkey_proj value_projb query_states key_states value_statesflatten_latent_dimrBattention_scoresattention_probsrrr__call__sR                zFlaxAttention.__call__)NT)__name__ __module__ __qualname____doc__r__annotations__rSrUrWfloatrXboolrYr float32rZrlryr{rrrrrrPzs       rPc@s|eZdZUdZeed<eed<eed<dZeed<dZe ed<e j Z e j ed <dZ e ed <dZe ed <d d ZdddZdS)FlaxBasicTransformerBlockae A Flax transformer block layer with `GLU` (Gated Linear Unit) activation function as described in: https://arxiv.org/abs/1706.03762 Parameters: dim (:obj:`int`): Inner hidden states dimension n_heads (:obj:`int`): Number of heads d_head (:obj:`int`): Hidden states dimension inside each head dropout (:obj:`float`, *optional*, defaults to 0.0): Dropout rate only_cross_attention (`bool`, defaults to `False`): Whether to only apply cross attention. dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): Parameters `dtype` use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): enable memory efficient attention https://arxiv.org/abs/2112.05682 split_head_dim (`bool`, *optional*, defaults to `False`): Whether to split the head dimension into a new axis for the self-attention computation. In most cases, enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. rwn_headsd_headrVrWFonly_cross_attentionrZrXrYc Cst|j|j|j|j|j|j|jd|_t|j|j|j|j|j|j|jd|_ t |j|j|jd|_ t j d|jd|_t j d|jd|_t j d|jd|_t j|jd|_dS)NrZ)rwrWrZh㈵>)epsilonrZra)rPrwrrrWrXrYrZattn1attn2FlaxFeedForwardffrd LayerNormnorm1norm2norm3rgrhrjrrrrls.  zFlaxBasicTransformerBlock.setupTcCs|}|jr|j||||d}n |j|||d}||}|}|j||||d}||}|}|j|||d}||}|j||dSNr~)rrrrrrrrh)rjrrrresidualrrrr2sz"FlaxBasicTransformerBlock.__call__NT)rrrrrrrWrrrr rrZrXrYrlrrrrrrs     rc@seZdZUdZeed<eed<eed<dZeed<dZeed<d Z e ed <d Z e ed <e j Ze jed <d Ze ed <d Ze ed<ddZdddZdS)FlaxTransformer2DModela A Spatial Transformer layer with Gated Linear Unit (GLU) activation function as described in: https://arxiv.org/pdf/1506.02025.pdf Parameters: in_channels (:obj:`int`): Input number of channels n_heads (:obj:`int`): Number of heads d_head (:obj:`int`): Hidden states dimension inside each head depth (:obj:`int`, *optional*, defaults to 1): Number of transformers block dropout (:obj:`float`, *optional*, defaults to 0.0): Dropout rate use_linear_projection (`bool`, defaults to `False`): tbd only_cross_attention (`bool`, defaults to `False`): tbd dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): Parameters `dtype` use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): enable memory efficient attention https://arxiv.org/abs/2112.05682 split_head_dim (`bool`, *optional*, defaults to `False`): Whether to split the head dimension into a new axis for the self-attention computation. In most cases, enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. in_channelsrrrodepthrVrWFuse_linear_projectionrrZrXrYcstjddd_jjjrtjjd_n tj dddjd_fdd t j D_ jrBtjjd_ n tj dddjd_ tjjd _dS) N r) num_groupsrr)roroVALID) kernel_sizestridespaddingrZc s2g|]}tjjjjjjjdqS))rWrrZrXrY)rrrrWrrZrXrY).0rCrkrjrr ~s z0FlaxTransformer2DModel.setup..ra)rd GroupNormnormrrrrerZproj_inConvrangertransformer_blocksproj_outrgrWrhrrrrrlos0   zFlaxTransformer2DModel.setupTc Cs|j\}}}}|}||}|jr |||||}||}n||}|||||}|jD] } | |||d}q1|jrL||}|||||}n |||||}||}||}|j||dSr)r'rrrqrrrrh) rjrrrbatchheightwidthchannelsrtransformer_blockrrrrs"      zFlaxTransformer2DModel.__call__Nr)rrrrrrrrWrrrrr rrZrXrYrlrrrrrrHs       *rc@HeZdZUdZeed<dZeed<ej Z ej ed<ddZ d d d Z d S) ra Flax module that encapsulates two Linear layers separated by a non-linearity. It is the counterpart of PyTorch's [`FeedForward`] class, with the following simplifications: - The activation function is currently hardcoded to a gated linear unit from: https://arxiv.org/abs/2002.05202 - `dim_out` is equal to `dim`. - The number of hidden dimensions is hardcoded to `dim * 4` in [`FlaxGELU`]. Parameters: dim (:obj:`int`): Inner hidden states dimension dropout (:obj:`float`, *optional*, defaults to 0.0): Dropout rate dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): Parameters `dtype` rwrVrWrZcCs,t|j|j|j|_tj|j|jd|_dS)Nr) FlaxGEGLUrwrWrZnet_0rdrenet_2rrrrrlszFlaxFeedForward.setupTcCs|j||d}||}|Sr)rr)rjrrrrrrs zFlaxFeedForward.__call__Nr rrrrrrrWrr rrZrlrrrrrrs  rc@r) ra Flax implementation of a Linear layer followed by the variant of the gated linear unit activation function from https://arxiv.org/abs/2002.05202. Parameters: dim (:obj:`int`): Input hidden states dimension dropout (:obj:`float`, *optional*, defaults to 0.0): Dropout rate dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): Parameters `dtype` rwrVrWrZcCs4|jd}tj|d|jd|_tj|jd|_dS)Nr}rnrra)rwrdrerZprojrgrWrhrirrrrls zFlaxGEGLU.setupTcCs6||}tj|ddd\}}|j|t||dS)Nrnr r~)rr splitrhrdgelu)rjrr hidden_linear hidden_gelurrrrs zFlaxGEGLU.__call__Nrrrrrrrs  r)r)r3rK flax.linenlinenrdr jax.numpynumpyr rrAr PrecisionHIGHESTrOModulerPrrrrrrrrs"   4 0zTj"