o
    ig                     @   s  d dl Z d dlmZ d dlmZmZmZ d dlZd dlm	Z	 d dl
m	  mZ ddlmZ ddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0 dd Z1ddddZ2dej3de4dej3fddZ5	dede	j6dej3d ej3d!ej3d"eej3 d#e7d$e7d%e$e& fd&d'Z8G d(d) d)e	j6Z9ed*G d+d, d,e	j6Z:G d-d. d.e	j6Z;G d/d0 d0eZ<G d1d2 d2e	j6Z=G d3d4 d4e	j6Z>G d5d6 d6e	j6Z?G d7d8 d8e	j6Z@G d9d: d:e	j6ZAG d;d< d<e	j6ZBG d=d> d>e	j6ZCG d?d@ d@e	j6ZDG dAdB dBe	j6ZEG dCdD dDe	j6ZFG dEdF dFe	jGZHG dGdH dHe	j6ZIG dIdJ dJe	j6ZJG dKdL dLe	j6ZKG dMdN dNe	j6ZLG dOdP dPe	j6ZMe'dQdRG dSdT dTe"ZNG dUdV dVZOe'G dWdX dXe"ZPG dYdZ dZe	j6ZQe'G d[d\ d\ePZRe'G d]d^ d^ePeZSG d_d` d`ePZTG dadb dbePeZUg dcZVdS )f    N)cached_property)CallableOptionalUnion   )ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)deprecate_kwarg)check_model_inputs   )
Emu3ConfigEmu3TextConfigEmu3VQVAEConfigc                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..N   dim)shapetorchcat)xx1x2 r(   Z/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/emu3/modeling_emu3.pyrotate_half/   s   r*   c                 C   sD   | |}| |}| | t| |  }|| t||  }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer*   )qkcossinposition_idsunsqueeze_dimq_embedk_embedr(   r(   r)   apply_rotary_pos_emb6   s
   

r4   hidden_statesn_repreturnc                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r"   expandreshape)r5   r6   batchnum_key_value_headsslenhead_dimr(   r(   r)   	repeat_kvQ   s
   0r>           modulequerykeyvalueattention_maskscalingdropoutkwargsc                 K   s   t || j}t || j}	t||dd| }
|d ur3|d d d d d d d |jd f }|
| }
tjj|
dtj	d
|j}
tjj|
|| jd}
t|
|	}|dd }||
fS )Nr   r   r   )r!   dtype)ptrainingr   )r>   num_key_value_groupsr#   matmul	transposer"   nn
functionalsoftmaxfloat32torI   rF   rK   
contiguous)r@   rA   rB   rC   rD   rE   rF   rG   
key_statesvalue_statesattn_weightscausal_maskattn_outputr(   r(   r)   eager_attention_forward]   s   
&rZ   c                       s   e Zd ZdZdedef fddZedddd		
	
ddej	de
ej	ej	f deej	 dee deej dee de
ej	ej	f fddZ  ZS )Emu3Attention=Multi-headed attention from 'Attention Is All You Need' paperconfig	layer_idxc                    s   t    || _|| _t|d|j|j | _|j|j | _	| jd | _
|j| _d| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j| j |j|jd| _d S )Nr=         Tbias)super__init__r]   r^   getattrhidden_sizenum_attention_headsr=   r;   rL   rE   attention_dropout	is_causalrO   Linearattention_biasq_projk_projv_projo_projselfr]   r^   	__class__r(   r)   rc   z   s(   
zEmu3Attention.__init__past_key_valuepast_key_values4.58new_nameversionNr5   position_embeddingsrD   cache_positionrG   r7   c                 K   s$  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}t|	|
||\}	}
|d urW|||d}||
|| j	|\}
}t
}| jjdkret| jj }|| |	|
||f| jsqdn| j| jd|\}}|jg |dR   }| |}||fS )Nr   r   r   )r/   r.   rz   eagerr?   )rF   rE   )r"   r=   rk   viewrN   rl   rm   r4   updater^   rZ   r]   _attn_implementationr   rK   rg   rE   r9   rT   rn   )rp   r5   ry   rD   rt   rz   rG   input_shapehidden_shapequery_statesrU   rV   r.   r/   cache_kwargsattention_interfacerY   rW   r(   r(   r)   forward   s8   


zEmu3Attention.forwardNN)__name__
__module____qualname____doc__r   intrc   r   r#   Tensortupler   r   
LongTensorr   r   r   __classcell__r(   r(   rq   r)   r[   w   s*    r[   RMSNormc                       s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	Emu3RMSNormư>c                    s&   t    tt|| _|| _dS )z:
        Emu3RMSNorm is equivalent to T5LayerNorm
        N)rb   rc   rO   	Parameterr#   onesweightvariance_epsilon)rp   re   epsrq   r(   r)   rc      s   

zEmu3RMSNorm.__init__c                 C   sJ   |j }|tj}|djddd}|t|| j  }| j|| S )Nr   r   T)keepdim)	rI   rS   r#   rR   powmeanrsqrtr   r   )rp   r5   input_dtypevariancer(   r(   r)   r      s
   zEmu3RMSNorm.forwardc                 C   s   t | jj d| j S )Nz, eps=)r   r   r"   r   rp   r(   r(   r)   
extra_repr   s   zEmu3RMSNorm.extra_repr)r   )r   r   r   rc   r   r   r   r(   r(   rq   r)   r      s    r   c                       $   e Zd Z fddZdd Z  ZS )Emu3MLPc                    sx   t    || _|j| _|j| _tj| j| j|jd| _tj| j| j|jd| _	tj| j| j|jd| _
t|j | _d S )Nr`   )rb   rc   r]   re   intermediate_sizerO   ri   mlp_bias	gate_projup_proj	down_projr   
hidden_actact_fnrp   r]   rq   r(   r)   rc      s   
zEmu3MLP.__init__c                 C   s$   |  | | || | }|S N)r   r   r   r   )rp   r%   r   r(   r(   r)   r      s    zEmu3MLP.forwardr   r   r   rc   r   r   r(   r(   rq   r)   r      s    
r   c                       s   e Zd Zdedef fddZedddd							
				ddejde	ej de	ej
 de	e de	e de	ej
 de	eejejf  dee dejfddZ  ZS )Emu3DecoderLayerr]   r^   c                    s`   t    |j| _t||d| _t|| _t|j|jd| _	t|j|jd| _
t|j| _d S )N)r]   r^   r   )rb   rc   re   r[   	self_attnr   mlpr   rms_norm_epsinput_layernormpost_attention_layernormrO   Dropoutrg   rF   ro   rq   r(   r)   rc      s   

zEmu3DecoderLayer.__init__rs   rt   ru   rv   NFr5   rD   r0   	use_cacherz   ry   rG   r7   c              
   K   sj   |}	|  |}| jd|||||||d|\}}
|	| | }|}	| |}| |}|	| | }|S )N)r5   rD   r0   rt   r   rz   ry   r(   )r   r   rF   r   r   )rp   r5   rD   r0   rt   r   rz   ry   rG   residual_r(   r(   r)   r      s&   




zEmu3DecoderLayer.forward)NNNFNN)r   r   r   r   r   rc   r   r#   r   r   r   r   boolr   r   r   r   r   r(   r(   rq   r)   r      s8    	
r   c                       s6   e Zd ZdZdef fddZdejfddZ  Z	S )Emu3VQVAEVectorQuantizera  
    A module for vector quantization using learned embedding vectors.

    This module implements the quantization process similar to te one described in
    the VQ-VAE (Vector Quantized Variational AutoEncoder) paper. It quantizes continuous
    input vectors into discrete codebook vectors, which are learned during training.
    Current implementation improves over previous ones by avoiding costly matrix multiplications
    and allowing for post-hoc remapping of indices.
    r]   c                    s>   t    t|j|j| _| jjj	d|j d|j  d S )Ng            ?)
rb   rc   rO   	Embeddingcodebook_size	embed_dim	embeddingr   datauniform_r   rq   r(   r)   rc     s   
"z!Emu3VQVAEVectorQuantizer.__init__hidden_statec                 C   s   |j \}}}}}|ddddd }|d|}tj|d ddd}tj| jjd dd	}	dt|| jj	dd }
||	 |
 }
tj
|
dd	}|||||}|S )
Nr   r   r      r   r   T)r!   r   r    )r"   permuterT   r|   r#   sumr   r   rM   rN   argmin)rp   r   
batch_sizetemporalchannelsheightwidthhidden_state_flattenedhidden_state_sumembedding_sum	distancesmin_encoding_indicesr(   r(   r)   r   !  s   z Emu3VQVAEVectorQuantizer.forward)
r   r   r   r   r   rc   r#   r   r   r   r(   r(   rq   r)   r     s    
r   c                       r   )Emu3VQVAEEncoderConvDownsamplec                    s$   t    tj||dddd| _d S )Nr   r   r   kernel_sizestridepaddingrb   rc   rO   Conv2dconvrp   in_channelsrq   r(   r)   rc   4     
z'Emu3VQVAEEncoderConvDownsample.__init__c                 C   s    t j|dddd}| |}|S )N)r   r   r   r   constantr   )padmoderC   )Fr   r   rp   r5   r(   r(   r)   r   8  s   
z&Emu3VQVAEEncoderConvDownsample.forwardr   r(   r(   rq   r)   r   3      r   c                       r   )Emu3VQVAEEncoderConvUpsamplec                    s$   t    tj||dddd| _d S )Nr   r   r   r   r   rq   r(   r)   rc   @  r   z%Emu3VQVAEEncoderConvUpsample.__init__c                 C   s   t j|ddd}| |}|S )N       @nearestscale_factorr   )r   interpolater   r   r(   r(   r)   r   D  s   
z$Emu3VQVAEEncoderConvUpsample.forwardr   r(   r(   rq   r)   r   ?  r   r   c                	       sF   e Zd Zdededee dee f fddZdejfdd	Z  Z	S )
Emu3VQVAEConv3d
in_channelout_channelr   r   c                    s   t    dd t|dd  |dd  D }d| _|d d d D ]}|  j|d |d  |d f7  _q!|  jd7  _tj||||d| _d S )	Nc                 S   s   g | ]\}}|| qS r(   r(   ).0
one_kernel
one_strider(   r(   r)   
<listcomp>T  s    z,Emu3VQVAEConv3d.__init__.<locals>.<listcomp>r   r(   r   r   )r   r   )r   )rb   rc   zipr   rO   Conv3dr   )rp   r   r   r   r   padding_sizespad_sizerq   r(   r)   rc   K  s   
$$zEmu3VQVAEConv3d.__init__r5   c                 C   s   t || j}| |}|S r   )r   r   r   r   r   r(   r(   r)   r   a  s   
zEmu3VQVAEConv3d.forward)
r   r   r   r   r   rc   r#   r   r   r   r(   r(   rq   r)   r   J  s    r   c                       s<   e Zd Zdedef fddZdejdejfddZ  ZS )	Emu3VQVAESpatialNormr   out_channelsc                    sN   t    tj|dddd| _tj||dddd| _tj||dddd| _d S )N    r   Tnum_channels
num_groupsr   affiner   r   r   )rb   rc   rO   	GroupNorm
norm_layerr   conv_yconv_brp   r   r   rq   r(   r)   rc   h  s*   
zEmu3VQVAESpatialNorm.__init__r5   quant_statesc                 C   s@   t j||jdd  dd}| |}|| | | | }|S )NrH   r   )sizer   )r   r   r"   r   r   r   )rp   r5   r   r(   r(   r)   r     s   
zEmu3VQVAESpatialNorm.forward	r   r   r   r   rc   r#   r   r   r   r(   r(   rq   r)   r   g  s    r   c                       6   e Zd Zdedef fddZdejfddZ  ZS )Emu3VQVAETemporalUpsampler   r   c                        t    t||ddd| _d S )Nr   r   r   r   r   r   r   r   rb   rc   r   r   rp   r   r   rq   r(   r)   rc        
z"Emu3VQVAETemporalUpsample.__init__r5   c                 C   sr   |j \}}}}}|ddddd |d|}tj|ddd	}|||||dddddd }| |}|S )
Nr   r   r   r   r   r   r   r   r   )r"   r   rT   r|   r   r   r   )rp   r5   r   r   r   r   r   r(   r(   r)   r     s    $
z!Emu3VQVAETemporalUpsample.forwardr   r(   r(   rq   r)   r         r   c                       r   )Emu3VQVAETemporalDownsampler   r   c                    r  )N)r   r   r   )r   r   r   r  r  r  rq   r(   r)   rc     r  z$Emu3VQVAETemporalDownsample.__init__r5   c                 C   s   |  |}|S r   )r   r   r(   r(   r)   r     s   
z#Emu3VQVAETemporalDownsample.forwardr   r(   r(   rq   r)   r	    r  r	  c                       s(   e Zd Z	d fdd	Zdd Z  ZS )Emu3VQVAETemporalResnetBlockNc                    s   t    || _|d u r|n|| _t|| _t||ddd| _t|| _	t||ddd| _
| j| jkrBtj||dddd| _d S d S )Nr  r  r  r   r   r   )rb   rc   r   r   rO   BatchNorm3dnorm1r   conv1norm2conv2r   nin_shortcutr   rq   r(   r)   rc     s4   
z%Emu3VQVAETemporalResnetBlock.__init__c                 C   sf   |}|  |}|t|9 }| |}| |}|t|9 }| |}| j| jkr/| |}|| S r   )	r  r#   sigmoidr  r  r  r   r   r  )rp   r5   r   r(   r(   r)   r     s   




z$Emu3VQVAETemporalResnetBlock.forwardr   r   r(   r(   rq   r)   r
    s     r
  c                       sT   e Zd Z		d
dedee dee f fddZddejdeej fdd	Z  Z	S )Emu3VQVAEResnetBlockNr   r   quant_channelsc                    s   t    || _|d u r|n|}|| _|| _|d u r/tj|dddd| _tj|dddd| _nt	||| _t	||| _tj
||dddd| _tj
||dddd| _| j| jkrdtj
||dddd| _d S d S )	Nr   r   Tr   r   r   r   r   )rb   rc   r   r   r  rO   r   r  r  r   r   r  r  r  )rp   r   r   r  rq   r(   r)   rc     sB   
zEmu3VQVAEResnetBlock.__init__r5   c                 C   s   | j d u rdn|f}|}| j|g|R  }|t|9 }| |}| j|g|R  }|t|9 }| |}| j| jkrA| 	|}|| S Nr(   )
r  r  r#   r  r  r  r  r   r   r  )rp   r5   r  	norm_argsr   r(   r(   r)   r     s   


zEmu3VQVAEResnetBlock.forwardr   r   )
r   r   r   r   r   rc   r#   r   r   r   r(   r(   rq   r)   r    s    $,r  c                
       sX   e Zd ZdZdef fddZ	ddejdeej de	ejeej f fd	d
Z
  ZS )Emu3VQVAEAttentionBlockr\   r]   c                    s   t    || _|j| _|j| _| j| j | _| j| j | jkr-td| j d| j d| jd | _	|j
| _d| _t| j| j| _t| j| j| _t| j| j| _t| j| j| _d| _d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r_   Fr   )rb   rc   r]   re   r   rf   	num_headsr=   
ValueErrorscalerg   rF   rh   rO   ri   rl   rm   rk   out_projrL   r   rq   r(   r)   rc   (  s&   


z Emu3VQVAEAttentionBlock.__init__Nr5   rD   r7   c              
   K   s   |j \}}}| |}| |}| |}	|||| j| jdd}|||| j| jdd}|	||| j| jdd}	t}
| j	j
dkrMt| j	j
 }
|
| |||	|| j| j| js\dn| jd\}}|||| }| |}||fS )z#Input shape: Batch x Time x Channelr   r   r{   r?   )rh   rE   rF   )r"   rk   rl   rm   r|   r  r=   rN   rZ   r]   r~   r   rh   r  rK   rF   r9   rT   r  )rp   r5   rD   rG   r   
seq_lengthr   querieskeysvaluesr   rY   rW   r(   r(   r)   r   ?  s.   




zEmu3VQVAEAttentionBlock.forwardr   )r   r   r   r   r   rc   r#   r   r   r   r   r   r(   r(   rq   r)   r  %  s    r  c                       s*   e Zd ZdZ fddZdddZ  ZS )Emu3VQVAEGroupNormz
    Same as the torch GroupNorm with the only difference that this ones accepts
    an optional kwarg `quant_states` which is not used. This class makes it easier to
    use SpatialNorm or GroupNorm without conditionals
    c                    s   t  jdi | d S r  )rb   rc   rp   rG   rq   r(   r)   rc   m  s   zEmu3VQVAEGroupNorm.__init__Nc                 C   s   t || j| j| j| jS r   )r   
group_normr   r   ra   r   )rp   inputr   r(   r(   r)   r   p  s   zEmu3VQVAEGroupNorm.forwardr   )r   r   r   r   rc   r   r   r(   r(   rq   r)   r  f  s    r  c                       s:   e Zd Zd fdd	Zddejdeej fddZ  ZS )	Emu3VQVAEMiddleBlockNc                    s`   t    t|||d| _t|| _|d u r t|dddd| _nt||| _t|||d| _	d S )Nr   r   r  r   r   Tr   )
rb   rc   r  block_1r  attn_1r  	attn_normr   block_2)rp   r]   r   r  rq   r(   r)   rc   u  s   

zEmu3VQVAEMiddleBlock.__init__r5   r   c                 C   s   |  ||}|}| ||}|j\}}}}||||| dd}| |d }|||||dddd}|| }| ||}|S )Nr   r   r   r   )	r%  r'  r"   r|   rN   r&  r9   r   r(  )rp   r5   r   r   r   r   r   r   r(   r(   r)   r     s   zEmu3VQVAEMiddleBlock.forwardr   )	r   r   r   rc   r#   FloatTensorr   r   r   r(   r(   rq   r)   r#  t  s    $r#  c                       ,   e Zd Z fddZdejfddZ  ZS )Emu3VQVAEDownBlockc              
      s(  t    t|j| _|j| _|j}|j}dt| }|| _t	
 | _t| jD ]i}t	
 }t	
 }t	
 }|||  }	|||  }
t| jD ]*}|t|	|
d |
}	|jd urq||jv rq|t| |t	j|	dddd qGt	 }||_||_||_|| jd krt|	|_| j| q(d S )Nr   r   r   r   r   Tr   r   )rb   rc   lenchannel_multipliernum_resolutionsnum_res_blocksbase_channelsr   in_channel_multiplierrO   
ModuleListdownrangeappendr  attn_resolutionsr  r   Moduleblockattn
attn_normsr   
downsample)rp   r]   r2  r/  r3  i_levelr:  r;  r<  block_in	block_outi_blockr5  rq   r(   r)   rc     sD   


zEmu3VQVAEDownBlock.__init__r5   c           
      C   s   t | jD ]^\}}t| jD ]H}|j| |}t|jdkrV|}|j| |}|j\}}}}	|	||||	 
dd}|j| |d }||||	|dddd}|| }q|| jd krc||}q|S )Nr   r   r   r   )	enumerater5  r6  r1  r:  r.  r;  r<  r"   r|   rN   r9   r   r0  r=  )
rp   r5   r>  blocksrA  r   r   r   r   r   r(   r(   r)   r     s    
zEmu3VQVAEDownBlock.forwardr   r   r   rc   r#   r)  r   r   r(   r(   rq   r)   r+    s    %r+  c                       s2   e Zd Z fddZdejdejfddZ  ZS )Emu3VQVAEUpBlockc              	      s  t    t|j| _|j| _|j}|j|jd  }t	 | _
tt| jD ]]}t	 }t	 }t	 }|j|j|  }t| jd D ]"}	|t|||d |}||jv re|t| |t|| qCt }
||
_||
_||
_|dkr|t||
_| j
d|
 q&d S )Nr   r   r$  r   )rb   rc   r.  r/  r0  r1  r   r2  rO   r4  upreversedr6  r7  r  r8  r  r   r9  r:  r;  r<  r   upsampleinsert)rp   r]   r  r?  r>  r:  r;  r<  r@  rA  rF  rq   r(   r)   rc     s@   



zEmu3VQVAEUpBlock.__init__r5   r   c                 C   s   t | jd d d D ]d\}}t| jd D ]J}|j| ||}t|jdkr_|}|j| ||}|j\}}}	}
|	|||	|
 
dd}|j| |d }|||	|
|dddd}|| }q|t| jd krn||}q
|S )Nr   r   r   r   r   )rB  rF  r6  r1  r:  r.  r;  r<  r"   r|   rN   r9   r   rH  )rp   r5   r   r>  rC  rA  r   r   r   r   r   r(   r(   r)   r     s    
zEmu3VQVAEUpBlock.forwardrD  r(   r(   rq   r)   rE    s    %rE  c                       r*  )Emu3VQVAEEncoderc                    s  t    |j}|j}|j}|j}|j}|rd| n|}||d  }tjj	||dddd| _
t|| _t||| _tjjd|ddd	| _tjj	||dddd| _tt|j}	t | _t | _t|	D ]}
t||}| j| qft|jD ]}t||d
}| j| qyd S )Nr   r   r   r   r   r   r   T)r   r   r   r   r-  )rb   rc   r2  r   double_latentlatent_channelsr/  r#   rO   r   conv_inr+  
down_blockr#  middle_blockr   norm_outconv_outr   mathlog2temporal_downsample_factorr4  	time_convtime_res_stackr6  r	  r7  r1  r
  )rp   r]   r2  r   rK  rL  r/  r   r?  temporal_down_blocksir   r   time_res_convrq   r(   r)   rc     s@   




zEmu3VQVAEEncoder.__init__pixel_valuesc                 C   s   |j d }|jdg|j dd  R  }| |}| |}| |}| |}|t|9 }| |}|jd|g|j dd  R  }|	ddddd}| j
D ]}||}|t|9 }qN| jD ]}||}q_|	ddddd}|S )Nr   r   r   r   r   r   )r"   r9   rM  rN  rO  rP  r#   r  rQ  r   rU  rV  )rp   rZ  temporal_dimr5   r   layerr(   r(   r)   r   3  s"   








zEmu3VQVAEEncoder.forward)r   r   r   rc   r#   r   r   r   r(   r(   rq   r)   rJ    s    'rJ  c                       s8   e Zd Zdef fddZdejdejfddZ  ZS )Emu3VQVAEDecoderr]   c           	         s  t    |j}|j|jd  }t | _t|j	D ]}t
|j|jd}| j| qtt|j}t | _t|D ]}t|j|j}| j| q<tj|j|dddd| _t|||d| _t|| _|j|jd  }t||| _tj||jdddd| _d S )Nr   r-  r   r   r   )r  r   )rb   rc   r   r2  r/  rO   r4  rV  r6  r1  r
  rL  r7  r   rR  rS  rT  rU  r   r   rM  r#  rO  rE  up_blockr   rP  r   rQ  )	rp   r]   r  r?  r   rY  temp_upsample_block_numrX  r   rq   r(   r)   rc   R  s@   



zEmu3VQVAEDecoder.__init__r5   r   c                 C   s  t j||fdd}|ddddd}| jD ]}||}q| jD ]}||}|t |9 }q|ddddd}t j|ddd\}}|jdg|jdd  R  }|jdg|jdd  R  }| 	|}| 
||}| ||}| ||}|t |9 }| |}|S )Nr   r    r   r   r   r   r   )r#   r$   r   rV  rU  r  chunkr9   r"   rM  rO  r^  rP  rQ  )rp   r5   r   hidden_quant_statesr\  r(   r(   r)   r   y  s$   




zEmu3VQVAEDecoder.forward)	r   r   r   r   rc   r#   r   r   r   r(   r(   rq   r)   r]  Q  s    'r]  aR  
    The VQ-VAE model used in Emu3 for encoding/decoding images into discrete tokens.
    This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from
    [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv
    Taigman](https://huggingface.co/papers/2203.13131).
    )custom_introc                       sz   e Zd ZU eed< dZdZdZdZdZ	dZ
g dZdd Zdef fdd	Zdejd
ejfddZdejfddZ  ZS )	Emu3VQVAEr]   
emuvideovqrZ  T)r
  r  r  r   c                 C   s\  t |tjtjfr6tjj|jddd |jd ur4tj|j\}}dt	
| }tj|j| | d S d S t |tjrqtjj|jt	
dd |jd urotj|j\}}|dkradt	
| nd}tj|j| | d S d S t |tjtjtjfrtj|jd tj|jd	 d S t |tjr|jj  |jd ur|jj|j   d S d S d S )
Nfan_outrelu)r   nonlinearityr      )ar   r   r?   )
isinstancerO   r   r   initkaiming_normal_r   ra   _calculate_fan_in_and_fan_outrR  sqrtr   ri   kaiming_uniform_BatchNorm2dr  r   	constant_r   r   normal_padding_idxzero_)rp   r@   fan_inr   boundr(   r(   r)   _init_weights  s.   


zEmu3VQVAE._init_weightsc                    s   t  | || _t|| _t|| _t|| _dt	|j
d  | _t|j|jddd| _t|j|jddd| _dt	|j
d  | _|   |   d S )Nr   r   )r   r   r   r  r  )rb   rc   r]   rJ  encoderr]  decoderr   quantizer.  r/  vision_spatial_factorr   rL  r   
quant_convpost_quant_convspatial_scale_factoreval	post_initr   rq   r(   r)   rc     s   


zEmu3VQVAE.__init__image_sizesc                    s   |j dk}|r jj}|j\}}}}|dd|ddd}n|j\}}}}} |}	|	ddddd}	 |	}	|	ddddd}	 	|	}
|rO|

dn|
} fddt||D }|S )Nr   r   r   r   r   c                    s@   g | ]\}}|d t |d  j d t |d  j f qS )Nr   r   )r   r{  )r   single_imager   r   r(   r)   r     s    .z$Emu3VQVAE.encode.<locals>.<listcomp>)ndimr]   rT  r"   r+   repeatrx  r   r|  rz  squeezer   )rp   rZ  r  is_imager   r   r   r   r   r5   codesimage_tokensr(   r   r)   encode  s    




zEmu3VQVAE.encoder5   c                 C   s   |j dk}|r|d}|j\}}}}| j| }|jd }||||||ddddd }| 	|}	|ddddd}|	ddddd}	| 
|	|}
|
||| jj | jj|| j || j }
|rn|
d d df S |
S )Nr   r   r   r   r   r   )r  r+   r"   rz  r   flattenr|   r   rT   r}  ry  r9   r]   rT  r   r~  )rp   r5   r  r   r   r   r   quantr   
post_quantvideor(   r(   r)   decode  s&   


$

zEmu3VQVAE.decode)r   r   r   r   __annotations__base_model_prefixmain_input_name_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backend_no_split_modulesrw  rc   r#   r   r  r  r   r(   r(   rq   r)   rc    s   
 	rc  c                   @   s   e Zd ZdZdd Zedd Zedd Zedd	 Zed
d Z	edd Z
edd Zdeej dejfddZdejdejfddZdS )Emu3ImageVocabularyMappingzM
    A class for mapping discrete image tokens from VQGAN to BPE tokens.
    c                 C   s"   || _ |d| _|d| _d S )Nz<|extra_200|>z<image>)	vocab_mapgeteol_token_idimage_token_id)rp   r  r(   r(   r)   rc     s   z#Emu3ImageVocabularyMapping.__init__c                 C      t dd | j D S )Nc                 S   s   g | ]\}}| d r|qS z<|visual token
startswithr   namevalr(   r(   r)   r         z;Emu3ImageVocabularyMapping.image_tokens.<locals>.<listcomp>sortedr  itemsr   r(   r(   r)   r       z'Emu3ImageVocabularyMapping.image_tokensc                 C   r  )Nc                 S   s   g | ]\}}| d r|qS r  r  r  r(   r(   r)   r     r  z?Emu3ImageVocabularyMapping.image_tokens_str.<locals>.<listcomp>r  r   r(   r(   r)   image_tokens_str  r  z+Emu3ImageVocabularyMapping.image_tokens_strc                    s    fdd j D S )Nc                    s$   i | ]}t |d d  j| qS )irH   )r   r  )r   tokenr   r(   r)   
<dictcomp>"  s   $ z6Emu3ImageVocabularyMapping.img2bpe.<locals>.<dictcomp>)r  r   r(   r   r)   img2bpe      z"Emu3ImageVocabularyMapping.img2bpec                 C   s   dd | j  D S )Nc                 S   s   i | ]\}}||qS r(   r(   )r   r-   vr(   r(   r)   r  &      z6Emu3ImageVocabularyMapping.bpe2img.<locals>.<dictcomp>)r  r  r   r(   r(   r)   bpe2img$  r  z"Emu3ImageVocabularyMapping.bpe2imgc                 C   >   t jt| j d t jd}| j D ]\}}|||< q|S Nr   rI   )r#   zerosmaxr  r  r   r  rp   mappingr-   r  r(   r(   r)   bpe2img_mapping_tensor(     
z1Emu3ImageVocabularyMapping.bpe2img_mapping_tensorc                 C   r  r  )r#   r  r  r  r  r   r  r  r(   r(   r)   img2bpe_mapping_tensor/  r  z1Emu3ImageVocabularyMapping.img2bpe_mapping_tensor	img_batchr7   c                 C   sR   |j }tj|jd dftjd| j }| j|d }tj||gdd}||S )Nr   r   r  cpur   r    )	devicer#   r   r"   r   r  r  rS   r$   )rp   r  r  eol_row
img_tokensr(   r(   r)   convert_img2bpe6  s
    
z*Emu3ImageVocabularyMapping.convert_img2bpec                 C   s0   |j }|dd df }| j|d }||S )N.r   r  )r  r  rS   )rp   r  r  r  r(   r(   r)   convert_bpe2img=  s   
z*Emu3ImageVocabularyMapping.convert_bpe2imgN)r   r   r   r   rc   r   r  r  r  r  r  r  listr#   r   r  r  r(   r(   r(   r)   r    s"    





r  c                   @   sD   e Zd ZU eed< dZdZdgZddgZdZ	dZ
dZdZdZdZdS )	Emu3PreTrainedModelr]   modelTr   rt   rX   FN)r   r   r   r   r  r  supports_gradient_checkpointingr  _skip_keys_device_placementr  r  _can_compile_fullgraph!_supports_param_buffer_assignmentr  r  r(   r(   r(   r)   r  D  s   
 r  c                       sD   e Zd ZU ejed< ddef fddZe e	dd Z
  ZS )	Emu3RotaryEmbeddinginv_freqNr]   c                    s   t    t|drt|jtr|jd|jd| _nd| _|j| _	|j| _
|| _t| j | _| | j|\}| _| jd|dd | j| _d S )Nrope_scaling	rope_typetypedefaultr  F)
persistent)rb   rc   hasattrrj  r  dictr  r  max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr]   r   rope_init_fnattention_scalingregister_bufferr  original_inv_freq)rp   r]   r  r  rq   r(   r)   rc   Y  s   
zEmu3RotaryEmbedding.__init__c           
      C   s   | j d d d d f  |jd dd|j}|d d d d d f  }t|jjtr6|jjdkr6|jjnd}t	j
|dd+ | |  dd}t	j||fdd	}| | j }| | j }	W d    n1 smw   Y  |j|jd
|	j|jd
fS )Nr   r   r   mpsr  F)device_typeenabledr   r    r  )r  floatr8   r"   rS   r  rj  r  strr#   autocastrN   r$   r.   r  r/   rI   )
rp   r%   r0   inv_freq_expandedposition_ids_expandedr  freqsembr.   r/   r(   r(   r)   r   j  s   0&zEmu3RotaryEmbedding.forwardr   )r   r   r   r#   r   r  r   rc   no_gradr   r   r   r(   r(   rq   r)   r  V  s   
 
r  c                       s   e Zd ZeedZdef fddZee								dde
ej de
ej de
ej d	e
e d
e
ej de
ej de
e dee defddZ  ZS )Emu3TextModel)r5   
attentionsr]   c                    s   t     j| _ j| _t j j| j| _t	 fddt
 jD | _t j jd| _t d| _d| _|   d S )Nc                    s   g | ]}t  |qS r(   )r   )r   r^   r]   r(   r)   r     r  z*Emu3TextModel.__init__.<locals>.<listcomp>r   r  F)rb   rc   pad_token_idrs  
vocab_sizerO   r   re   embed_tokensr4  r6  num_hidden_layerslayersr   r   normr  
rotary_embgradient_checkpointingr  r   rq   r  r)   rc     s   zEmu3TextModel.__init__N	input_idsrD   r0   rt   inputs_embedsrz   r   rG   r7   c              	   K   s   |d u |d uA rt d|d u r| |}|r!|d u r!t| jd}|d u r=|d ur-| nd}	tj|	|	|jd  |jd}|d u rF|	d}t
| j|||||d}
|}| ||}| jd | jj D ]}||f|
||||d|}qb| |}t||dS )	Nz:You must specify exactly one of input_ids or inputs_embedsr  r   r   )r  )r]   input_embedsrD   rz   rt   r0   )rD   r0   rt   rz   ry   )last_hidden_statert   )r  r  r	   r]   get_seq_lengthr#   aranger"   r  r+   r   r  r  r  r  r   )rp   r  rD   r0   rt   r  rz   r   rG   past_seen_tokensrX   r5   ry   decoder_layerr(   r(   r)   r     sP   

	

zEmu3TextModel.forward)NNNNNNN)r   r   r   r   r[   _can_record_outputsr   rc   r   r   r   r#   r   r   r   r)  r   r   r   r   r   r   r(   r(   rq   r)   r  z  sB    	
r  c                       s   e Zd ZU dgZddiZddgdgfiZeed<  fddZe	e
																	
ddeej deej deej dee deej deej dee deej deeejf dee defddZ  ZS )Emu3ForCausalLMlm_head.weightlm_headcolwise_repr5   logitsr]   c                    s@   t  | t|| _|j| _tj|j|jdd| _| 	  d S NFr`   )
rb   rc   r  r  r  rO   ri   re   r  r  r   rq   r(   r)   rc     s
   
zEmu3ForCausalLM.__init__Nr   r  rD   r0   rt   r  labelsr   rz   logits_to_keeprG   r7   c
              
   K   s   | j d|||||||d|
}|j}t|	trt|	 dn|	}| |dd|ddf }d}|durB| jd||| jjd|
}t	|||j
|j|jdS )a  
        Example:

        ```python
        >>> from transformers import Emu3Processor, Emu3ForConditionalGeneration
        >>> import torch
        >>> import requests
        >>> from PIL import Image

        >>> model = Emu3ForCausalLM.from_pretrained("BAAI/Emu3-Chat-hf", dtype=torch.bfloat16)
        >>> processor = Emu3Processor.from_pretrained("BAAI/Emu3-Chat-hf")

        >>> inputs = processor(text=["Can you write me a poem about winter."], return_tensors="pt").to(model.device)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```r  rD   r0   rt   r  r   rz   Nr  r  r  lossr  rt   r5   r  r(   )r  r  rj  r   slicer  loss_functionr]   r  r   rt   r5   r  )rp   r  rD   r0   rt   r  r  r   rz   r  rG   outputsr5   slice_indicesr  r  r(   r(   r)   r     s0    zEmu3ForCausalLM.forward)	NNNNNNNNr   )r   r   r   _tied_weights_keys_tp_plan_pp_planr   r  rc   r   r   r   r#   r   r   r   r)  r   r   r   r   r   r   r   r   r(   r(   rq   r)   r    sP   
 		
r  c                       s:  e Zd ZddiZ fddZdd Zdd Zd	d
 Zdd Zde	j
de	jfddZde	j
de	jfddZe	jde	jdedefddZde	jde	j
de	j
fddZee									d'dee	j dee	j
 dee	j dee	j dee	j d ee dee	j
 d!ee d"ee	j d#ee d$eeef fd%d&Z  ZS )(	Emu3Modelztext_model.model
text_modelc                    s>   t  | t|j| _t|j| _t	|j
| _|   d S r   )rb   rc   r  _from_configtext_configr  rc  	vq_configvqmodelr  vocabulary_mapvocabulary_mappingr  r   rq   r(   r)   rc     s
   zEmu3Model.__init__c                 C   
   | j  S r   )r  get_input_embeddingsr   r(   r(   r)   r  '     
zEmu3Model.get_input_embeddingsc                 C      | j | d S r   )r  set_input_embeddingsrp   rC   r(   r(   r)   r  *     zEmu3Model.set_input_embeddingsc                 C   s
   || _ d S r   r  rp   ry  r(   r(   r)   set_decoder-  r  zEmu3Model.set_decoderc                 C      | j S r   r  r   r(   r(   r)   get_decoder0     zEmu3Model.get_decoderrZ  r  c                    s.    j ||} fdd|D }t|}|S )a  
        Tokenizes images into discrete tokens with VQGAN module. Converts
        obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
        special tokens.

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                The tensors corresponding to the input images.
            image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
                The sizes of the images in the batch, being (height, width) for each image.
        c                    s   g | ]
} j | qS r(   )r  r  r  )r   tokensr   r(   r)   r   @  s    z.Emu3Model.get_image_tokens.<locals>.<listcomp>)r  r  r#   r$   )rp   rZ  r  image_tokens_listbpe_tokens_list
bpe_tokensr(   r   r)   get_image_tokens3  s   
zEmu3Model.get_image_tokensc                    s:     ||} fdd|D }  |}t||}|S )a7  
        Tokenizes images into discrete tokens with VQGAN module and embeds
        them with text embeddings layer

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
                The tensors corresponding to the input images.
        c                    s,   g | ]\}}| j j | j j d   qS r,  )r  r{  )r   r   r   r   r(   r)   r   N  s    z0Emu3Model.get_image_features.<locals>.<listcomp>)r#  r  r#   split)rp   rZ  r  r  split_sizesimage_featuresr(   r   r)   get_image_featuresD  s   	
zEmu3Model.get_image_featuresr  r   r   c                 C   s>   |ddddf  d||d }| j|}| j|}|S )a  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.

        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
            height (`int`):
                Height of the generated image before upsampling.
            width (`int`):
                Width of the generated image before upsampling.
        Nr   r   )r|   r  r  r  r  )rp   r  r   r   	sequencesimager(   r(   r)   decode_image_tokensV  s   "zEmu3Model.decode_image_tokensr  r  r&  c                 C   s   |du r||   tj| jjtj|jdk}|d}n|| jjk}| }|	d
||j}|jd |jd  }||  | krPtd| d| |S )z
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        N)rI   r  r   r   r   z6Image features and image tokens do not match: tokens: z, features )r  r#   tensorr  r  longr  allr   r+   	expand_asrS   r"   numelr  )rp   r  r  r&  special_image_maskn_image_tokensn_image_featuresr(   r(   r)   get_placeholder_maski  s   zEmu3Model.get_placeholder_maskNrD   r0   rt   r   rz   rG   r7   c
              	   K   s   |du |duA rt d|du r|  |}|dur5| ||}tj|dd}| j|||d}|||}| jd||||||	d|
}|S )ap  
        image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
            The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using
            [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses
            [`Emu3ImageProcessor`] for processing images).
        NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either oner   r    )r  r&  )rD   r0   rt   r  r   rz   r(   )r  r  r'  r#   r$   r4  masked_scatterr  )rp   r  rZ  r  rD   r0   rt   r  r   rz   rG   image_embedsr1  r  r(   r(   r)   r     s0   
zEmu3Model.forward)	NNNNNNNNN)r   r   r   _checkpoint_conversion_mappingrc   r  r  r  r  r#   r)  r   r#  r'  r  r   r+  r4  r   r   r   r   r   r   r   r   r   r   r   r   r   r(   r(   rq   r)   r
    sh    	
	

r
  c                       sV  e Zd ZdZdgZddddZ fddZd	d
 Zdd Zde	j
fddZdd Zdd Zedd Zedd Zedd Zdd Zee											d/deej deej d eej d!eej d"eej d#ee d$eej d%ee d&eej d'eej d(eeejf d)ee dee e!f fd*d+Z"						,	d0 fd-d.	Z#  Z$S )1Emu3ForConditionalGeneration r  zmodel.text_modelzmodel.vqmodelr  )z^text_model.modelz^vqmodelz^text_model.lm_headc                    s<   t  | t|| _tj|jj|jjdd| _	| 
  d S r  )rb   rc   r
  r  rO   ri   r  re   r  r  r  r   rq   r(   r)   rc     s   
z%Emu3ForConditionalGeneration.__init__c                 C   r  r   )r  r  r   r(   r(   r)   r    r  z1Emu3ForConditionalGeneration.get_input_embeddingsc                 C   r  r   )r  r  r  r(   r(   r)   r    r  z1Emu3ForConditionalGeneration.set_input_embeddingsr7   c                 C   r  r   )r  r   r(   r(   r)   get_output_embeddings  r  z2Emu3ForConditionalGeneration.get_output_embeddingsc                 C   r  r   )r  r  r  r(   r(   r)   r    r  z(Emu3ForConditionalGeneration.set_decoderc                 C   r  r   )r  r  r   r(   r(   r)   r    r  z(Emu3ForConditionalGeneration.get_decoderc                 C      | j jS r   )r  r  r   r(   r(   r)   r       z'Emu3ForConditionalGeneration.text_modelc                 C   r;  r   )r  r  r   r(   r(   r)   r    r<  z$Emu3ForConditionalGeneration.vqmodelc                 C   r;  r   )r  r  r   r(   r(   r)   r    r<  z/Emu3ForConditionalGeneration.vocabulary_mappingc                 K   s   | j jdi |S r  )r  r+  r   r(   r(   r)   r+    s   z0Emu3ForConditionalGeneration.decode_image_tokensNr   r  rZ  r  rD   r0   rt   r  r   rz   r  r  rG   c              
   K   s   | j d|||||||	d|}|d }t|trt| dn|}| |dd|ddf }d}|
durD| jd||
| jjjd|}t	|||j
|j|jdS )an  
        image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
            The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using
            [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses
            [`Emu3ImageProcessor`] for processing images).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import Emu3Processor, Emu3ForConditionalGeneration
        >>> import torch
        >>> import requests
        >>> from PIL import Image

        >>> model = Emu3ForConditionalGeneration.from_pretrained("BAAI/Emu3-Chat-hf", dtype=torch.bfloat16)
        >>> processor = Emu3Processor.from_pretrained("BAAI/Emu3-Chat-hf")

        >>> conversation = [
        ...     {
        ...     "role": "system",
        ...     "content": [
        ...         {"type": "text", "text": "You are a helpful assistant."},
        ...         ],
        ...     },
        ...     {
        ...     "role": "user",
        ...     "content": [
        ...         {"type": "image"},
        ...         {"type": "text", "text": "Please describe the image."},
        ...         ],
        ...     },
        ... ]

        >>> prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
        >>> image = Image.open(requests.get("https://www.ilankelman.org/stopsigns/australia.jpg", stream=True).raw)

        >>> inputs = processor(images=[image], text=[prompt], return_tensors="pt").to(model.device, torch.bfloat16)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```r  r   Nr   r  r(   )r  rj  r   r  r  r  r]   r  r  r   rt   r5   r  )rp   r  rZ  r  rD   r0   rt   r  r   rz   r  r  rG   r  r5   r  r  r  r(   r(   r)   r     s8   >z$Emu3ForConditionalGeneration.forwardTc	              
      s<   t  j|f|||||||d|	}
|d dkrd |
d< |
S )N)rt   rD   r  rz   r0   rZ  r   r   rZ  )rb   prepare_inputs_for_generation)rp   r  rt   rD   r  rz   r0   r   rZ  rG   model_inputsrq   r(   r)   r=  ?  s    	z:Emu3ForConditionalGeneration.prepare_inputs_for_generation)NNNNNNNNNNr   )NNNNNTN)%r   r   r   r  r  r7  rc   r  r  rO   r9  r:  r  r  propertyr  r  r  r+  r   r   r   r#   r   r)  r   r   r   r   r   r   r   r   r   r   r=  r   r(   r(   rq   r)   r8    s    


	

]r8  )r8  r  r  r  rc  r
  )Nr   )r?   )WrR  	functoolsr   typingr   r   r   r#   torch.nnrO   torch.nn.functionalrP   r   activationsr   cache_utilsr   r	   
generationr
   integrationsr   masking_utilsr   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.deprecationr   utils.genericr   configuration_emu3r   r   r   r*   r4   r   r   r>   r9  r  rZ   r[   r   r   r   r   r   r   r   r   r   r	  r
  r  r  r   r  r#  r+  rE  rJ  r]  rc  r  r  r  r  r  r
  r8  __all__r(   r(   r(   r)   <module>   s   

G."$1?A";:FFo6$SL  ,