o
    eiZ                     @   sD  d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlmZ	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddlmZm Z  ddl!m"Z" ddl#m$Z$m%Z% eeddG dd deZ&G dd dej'Z(G dd dej'Z)G dd dej'Z*dd Z+eddBdd Z,d!ej-d"e.d#ej-fd$d%Z/	&dCd'ej'd(ej-d)ej-d*ej-d+ej-dB d,e0d-e0d.ee fd/d0Z1ee,G d1d2 d2ej'Z2G d3d4 d4ej'Z3G d5d6 d6eZ4eG d7d8 d8eZ5ed9dG d:d; d;e5Z6eG d<d= d=eZ7ed>dG d?d@ d@e5Z8g dAZ9dS )D    N)Callable)	dataclass)nn   )initialization)ACT2FN)use_kernel_func_from_hubuse_kernelized_func)GradientCheckpointingLayer)BaseModelOutputCausalLMOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuple)maybe_autocastmerge_with_config_defaults)capture_outputs   )ParakeetCTCConfigParakeetEncoderConfigz
    Extends [~modeling_outputs.BaseModelOutput] to include the output attention mask since sequence length is not preserved in the model's forward.
    )custom_introc                   @   s    e Zd ZU dZejdB ed< dS )ParakeetEncoderModelOutputNattention_mask)__name__
__module____qualname__r   torchTensor__annotations__ r#   r#   l/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/parakeet/modeling_parakeet.pyr   )   s   
 r   c                       sL   e Zd ZU dZejed< d
def fddZe	 dejfdd	Z
  ZS )$ParakeetEncoderRelPositionalEncodingz*Relative positional encoding for Parakeet.inv_freqNconfigc                    sZ   t    |j| _d}d|tjd|jdtjdj|tjd|j   }| j	d|dd	 d S )
N     @      ?r      dtype)devicer,   r&   F)
persistent)
super__init__max_position_embeddingsr    arangehidden_sizeint64tofloatregister_buffer)selfr'   r-   baser&   	__class__r#   r$   r0   8   s   
 z-ParakeetEncoderRelPositionalEncoding.__init__hidden_statesc                 C   sD  |j d }|| jkrtd| d| j dtj|d | d|jd}| jd d d d f  |j d dd	|j}|d d d d f  }t
|jjtrW|jjdkrW|jjnd	}t|d
d4 | |  dd}| }| }	tj||	gdd}
|
jg |
j d d dR  }
W d    n1 sw   Y  |
j	|jdS )Nr   zSequence Length: z= has to be less or equal than config.max_position_embeddings .r-   r   mpscpuF)device_typeenabledr*   dimr+   )shaper1   
ValueErrorr    r2   r-   r&   r6   expandr5   
isinstancetypestrr   	transposesincosstackreshaper,   )r8   r<   
seq_lengthposition_idsinv_freq_expandedposition_ids_expandedrB   freqsrN   rO   	pos_embedr#   r#   r$   forwardF   s2   

. z,ParakeetEncoderRelPositionalEncoding.forwardN)r   r   r   __doc__r    r!   r"   r   r0   no_gradrX   __classcell__r#   r#   r:   r$   r%   3   s   
 
r%   c                       s*   e Zd Zdef fddZdd Z  ZS )ParakeetEncoderFeedForwardr'   c                    sR   t    tj|j|j|jd| _t|j	 | _
tj|j|j|jd| _|j| _d S )Nbias)r/   r0   r   Linearr3   intermediate_sizeattention_biaslinear1r   
hidden_act
activationlinear2activation_dropoutr8   r'   r:   r#   r$   r0   f   s
   
z#ParakeetEncoderFeedForward.__init__c                 C   s4   |  | |}tjj|| j| jd}| |}|S )Nptraining)re   rc   r   
functionaldropoutrg   rk   rf   )r8   r<   r#   r#   r$   rX   m   s   
z"ParakeetEncoderFeedForward.forwardr   r   r   r   r0   rX   r\   r#   r#   r:   r$   r]   e   s    r]   c                       s.   e Zd Zddef fddZdddZ  ZS ) ParakeetEncoderConvolutionModuleNr'   c              	      s   t    |j}|du r|j}tt|dd | _n|d }t|dd | _|d d | _t	j
|d| ddd|jd	| _t	j
|||d| j||jd
| _t	|| _t	j
||ddd|jd	| _dS )z
        Args:
            config (ParakeetEncoderConfig): Configuration for the model.
            module_config (dict): Configuration for the module (e.g., encoder or decoder).
        Nrd   silukernel_sizere   r   r*   r   )rq   stridepaddingr_   )rr   rs   groupsr_   )r/   r0   r3   conv_kernel_sizer   getattrre   getrs   r   Conv1dconvolution_biaspointwise_conv1depthwise_convBatchNorm1dnormpointwise_conv2)r8   r'   module_configchannelsrq   r:   r#   r$   r0   u   s0   
	z)ParakeetEncoderConvolutionModule.__init__c                 C   s   | dd}| |}tjj|dd}|dur6|jtjkr&tj| dd}n
tj|dk dd}|	|d}| 
|}| |}| |}| |}| ddS )aY  
        Compute convolution module.

        Args:
            hidden_states (`torch.Tensor` of shape `(batch, time, channels)`): Input tensor.
            attention_mask (`torch.Tensor` of shape `(batch, 1, time, time)`): Attention mask.

        Returns:
            `torch.Tensor`: Output tensor of shape `(batch, time, channels)`.

        r   r*   rD   N        )rM   rz   r   rl   glur,   r    boolallmasked_fillr{   r}   re   r~   )r8   r<   r   all_masked_rowsr#   r#   r$   rX      s   




z(ParakeetEncoderConvolutionModule.forwardrY   rn   r#   r#   r:   r$   ro   t   s    "ro   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..Nr>   r*   rD   )rG   r    cat)xx1x2r#   r#   r$   rotate_half   s   r   rotary_pos_embc                 C   sD   | |}| |}| | t| |  }|| t||  }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkrO   rN   unsqueeze_dimq_embedk_embedr#   r#   r$   apply_rotary_pos_emb   s
   

r   r<   n_repreturnc                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rG   rI   rQ   )r<   r   batchnum_key_value_headsslenhead_dimr#   r#   r$   	repeat_kv   s
   0r   r   modulequerykeyvaluer   scalingrm   kwargsc                 K   s   t || j}t || j}	t||dd| }
|d ur |
| }
tjj|
dtjd	|j
}
tjj|
|| jd}
t|
|	}|dd }||
fS )Nr*   r   r>   rE   r,   ri   r   )r   num_key_value_groupsr    matmulrM   r   rl   softmaxfloat32r5   r,   rm   rk   
contiguous)r   r   r   r   r   r   rm   r   
key_statesvalue_statesattn_weightsattn_outputr#   r#   r$   eager_attention_forward   s   
r   c                       sr   e Zd ZdZdedef fddZ	ddejdejdB d	ejdB d
e	e
 deejejf f
ddZdd Z  ZS )ParakeetEncoderAttentionztMulti-head attention with relative positional encoding. See section 3.3 of https://huggingface.co/papers/1901.02860.r'   	layer_idxc                    s  t    || _|| _t|d|j|j | _|j|j | _	| jd | _
|j| _d| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j| j |j|jd| _tj|j|j| j dd| _tt|j| j| _tt|j| j| _d S )Nr   g      Fr^   )r/   r0   r'   r   rv   r3   num_attention_headsr   r   r   r   attention_dropout	is_causalr   r`   rb   q_projk_projv_projo_projrelative_k_proj	Parameterr    zerosbias_ubias_vr8   r'   r   r:   r#   r$   r0     s.   
z!ParakeetEncoderAttention.__init__Nr<   position_embeddingsr   r   r   c              	   K   s  |j d d }|\}}||d| jf}| ||dd}	| ||dd}
| ||dd}t| j	j
t}|	| jd| j	jd| j }|	| jd| j	jd| j }| |}||d| j	j| j}||dddd }| |}|dd |f }|| j }|d ur|| td}|| f||
||| jsdn| j| jd	|\}}|jg |dR   }| |}||fS )
Nr>   r   r*   r   r   .z-infr   )r   r   r   r   rm   r   )rG   r   r   viewrM   r   r   r   get_interfacer'   _attn_implementationr   r   r   r   r   permute
_rel_shiftr   masked_fill_logical_notr6   rk   r   rQ   r   r   )r8   r<   r   r   r   input_shape
batch_sizerR   hidden_shapequery_statesr   r   attention_interfacequery_states_with_bias_uquery_states_with_bias_vrelative_key_states	matrix_bdr   r   r#   r#   r$   rX   #  sL   




z ParakeetEncoderAttention.forwardc                 C   sX   |j \}}}}tjj|dd}|||d|}|ddddddf ||||}|S )ztRelative position shift for Shaw et al. style attention. See appendix B of https://huggingface.co/papers/1901.02860.)r   r   )padr>   Nr   )rG   r   rl   r   r   )r8   attention_scoresr   	num_headsquery_lengthposition_lengthr#   r#   r$   r   \  s
   &z#ParakeetEncoderAttention._rel_shiftrY   )r   r   r   rZ   r   intr0   r    r!   r   r   tuplerX   r   r\   r#   r#   r:   r$   r     s     !
9r   c                       sP   e Zd Zdef fddZdejdejfddZ	dd	ejd
ejfddZ
  ZS ) ParakeetEncoderSubsamplingConv2Dr'   c                    s  t    |j| _|j| _|j| _| jd d | _t	t
|j| _t | _| jtjd| j| j| j| jd | jt  t| jd D ]-}| jtj| j| j| j| j| j| jd | jtj| j| jdd | jt  qH|j| j| j  }tj|j| |jdd| _d S )Nr   r*   )rq   rr   rs   )rq   rr   rs   rt   rq   Tr^   )r/   r0   subsampling_conv_kernel_sizerq   subsampling_conv_striderr   subsampling_conv_channelsr   rs   r   mathlog2subsampling_factor
num_layersr   
ModuleListlayersappendConv2dReLUrangenum_mel_binsr`   r3   linear)r8   r'   i
out_lengthr:   r#   r$   r0   f  s4   

z)ParakeetEncoderSubsamplingConv2D.__init__input_lengths
conv_layerc                 C   sV   t |dr)|jdkr)|j}|jd }|jd }||d  |d  | | d }|S |S )Nrr   )r   r   r   r   )hasattrrr   rs   rq   )r8   r   r   rs   rq   rr   output_lengthsr#   r#   r$   _get_output_length  s   

 z3ParakeetEncoderSubsamplingConv2D._get_output_lengthNinput_featuresr   c                 C   s   | d}|d ur|dnd }| jD ]9}||}t|tjrL|d urL| ||}|jd }tj	||j
d|d d d f k }||d d d d d d f 9 }q|dd|jd |jd d}| |}|S )Nr   r>   r*   r?   r   )r   sumr   rJ   r   r   r   rG   r    r2   r-   rM   rQ   r   )r8   r   r   r<   current_lengthslayercurrent_seq_lengthchannel_maskr#   r#   r$   rX     s   


"
z(ParakeetEncoderSubsamplingConv2D.forwardrY   )r   r   r   r   r0   r    r!   r   r   r   rX   r\   r#   r#   r:   r$   r   e  s    # r   c                       sd   e Zd ZddededB f fddZ		ddejdejdB dejdB d	ee	 d
ejf
ddZ
  ZS )ParakeetEncoderBlockNr'   r   c                    s   t    d| _t|| _t||| _t|| _t|| _	t
|j| _t
|j| _t
|j| _t
|j| _t
|j| _d S NF)r/   r0   gradient_checkpointingr]   feed_forward1r   	self_attnro   convfeed_forward2r   	LayerNormr3   norm_feed_forward1norm_self_att	norm_convnorm_feed_forward2norm_outr   r:   r#   r$   r0     s   



zParakeetEncoderBlock.__init__r<   r   r   r   r   c                 K   s   |}|  | |}|d|  }| |}| jd|||d|\}}|| }| j| ||d}	||	 }| | |}
|d|
  }| |}|S )Ng      ?)r<   r   r   )r   r#   )	r   r   r   r   r   r   r   r   r   )r8   r<   r   r   r   residualnormalized_hidden_statesr   _conv_output
ff2_outputr#   r#   r$   rX     s$   


zParakeetEncoderBlock.forwardrY   NN)r   r   r   r   r   r0   r    r!   r   r   rX   r\   r#   r#   r:   r$   r     s    r   c                       s   e Zd ZU eed< dZdZdZdZdgZ	dZ
dZdZdZdZdZeedZe  fd	d
ZdejfddZddejdedB fddZ  ZS )ParakeetPreTrainedModelr'   modelr   audioTr   F)r<   
attentionsc                    s   t  | t| jdr| jj}n	t| j dd}t|tr3t	j
|jd|d t	j
|jd|d d S t|trUddtjd| jjdtjd	| jj   }t	|j| d S d S )
Ninitializer_rangeg{Gz?r   )meanstdr)   r(   r   r*   r+   )r/   _init_weightsr   r'   r  rv   get_text_configrJ   r   initnormal_r   r   r%   r    r2   r3   r4   copy_r&   )r8   r   r
  r&   r:   r#   r$   r    s   


"z%ParakeetPreTrainedModel._init_weightsr   c           
      C   s   t | jtr
| jjn| j}|j}|j}tt|j	}|d d d }|| }|}t
|D ]}	t|jtjd| |d }t|}q-|jtjdS )Nr   r*   r+   r)   )rJ   r'   r   encoder_configr   r   r   r   r   r   r   r    divr5   r6   floor)
r8   r   r  rq   rr   r   all_paddingsadd_padlengthsr   r#   r#   r$   _get_subsampling_output_length  s   z6ParakeetPreTrainedModel._get_subsampling_output_lengthNr   target_lengthc                 C   sH   |  |d}|dur|n| }tj||jd|dddf k }|S )z
        Convert the input attention mask to its subsampled form. `target_length` sets the desired output length, useful
        when the attention mask length differs from `sum(-1).max()` (i.e., when the longest sequence in the batch is padded)
        r>   Nr?   )r  r   maxr    r2   r-   )r8   r   r  r   
max_lengthr#   r#   r$   _get_output_attention_mask  s    z2ParakeetPreTrainedModel._get_output_attention_maskrY   )r   r   r   r   r"   base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_flat_attention_mask_supports_sdpa_supports_flex_attn_supports_flash_attn_can_compile_fullgraph_supports_attention_backendr   r   _can_record_outputsr    r[   r  r!   r  r   r  r\   r#   r#   r:   r$   r    s(   
 "r  z{
    The Parakeet Encoder model, based on the [Fast Conformer architecture](https://huggingface.co/papers/2305.05084).
    c                       st   e Zd ZU eed< dZdef fddZeee	e
		ddejdejdB dedB d	ee d
ef
ddZ  ZS )ParakeetEncoderr'   encoderc                    s   t     | _d| _ j| _ j| _ j| _ jr!t	 j
nd| _t | _t | _t fddt jD | _|   d S )NFr)   c                    s   g | ]}t  |qS r#   )r   ).0r   r'   r#   r$   
<listcomp>7  s    z,ParakeetEncoder.__init__.<locals>.<listcomp>)r/   r0   r'   r   rm   dropout_positions	layerdropscale_inputr   sqrtr3   input_scaler   subsamplingr%   encode_positionsr   r   r   num_hidden_layersr   	post_initrh   r:   r*  r$   r0   )  s   

zParakeetEncoder.__init__Nr   r   output_attention_maskr   r   c                 K   s   |  ||}|| j }| |}tjj|| j| jd}tjj|| j| jd}|durN| j||j	d d}|
dd|j	d d}||dd@ }|
d}| jD ] }d}	| jrdtg }
|
| jk rdd}	|	sq||f||d	|}qQt||r|| d
S dd
S )aJ  
        output_attention_mask (`bool`, *optional*):
            Whether to return the output attention mask.

        Example:

        ```python
        >>> from transformers import AutoProcessor, ParakeetEncoder
        >>> from datasets import load_dataset, Audio

        >>> model_id = "nvidia/parakeet-ctc-1.1b"
        >>> processor = AutoProcessor.from_pretrained(model_id)
        >>> encoder = ParakeetEncoder.from_pretrained(model_id)

        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate))

        >>> inputs = processor(ds[0]["audio"]["array"])
        >>> encoder_outputs = encoder(**inputs)

        >>> print(encoder_outputs.last_hidden_state.shape)
        ```
        ri   Nr   r  r>   r*   FT)r   r   )last_hidden_stater   )r1  r0  r2  r   rl   rm   rk   r,  r  rG   r   rI   rM   r   r    randr-  r   r   )r8   r   r   r5  r   r<   r   output_maskencoder_layerto_dropdropout_probabilityr#   r#   r$   rX   <  sB   #






zParakeetEncoder.forwardr  )r   r   r   r   r"   r  r0   r   r   r   r   r    r!   r   r   r   r   rX   r\   r#   r#   r:   r$   r'     s*   
 r'  c                   @   sf   e Zd ZU dZejed< dZeej	 dB ed< dZ
eeej	  dB ed< dZeeej	  dB ed< dS )ParakeetGenerateOutputal  
    Outputs of Parakeet models.

    Args:
        sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
            if all batches finished early due to the `eos_token_id`.
        logits (`tuple(torch.FloatTensor)` *optional*, returned when `output_logits=True`):
            Unprocessed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
            at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for
            each generated token), with each tensor of shape `(batch_size, config.vocab_size)`.
        attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True`):
            Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
            `torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
        hidden_states (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_hidden_states=True`):
            Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
            `torch.FloatTensor` of shape `(batch_size, generated_length, hidden_size)`.
    	sequencesNlogitsr  r<   )r   r   r   rZ   r    
LongTensorr"   r?  r   FloatTensorr  r<   r#   r#   r#   r$   r=    s   
 
r=  zS
    Parakeet Encoder with a Connectionist Temporal Classification (CTC) head.
    c                       s   e Zd ZU eed< def fddZee		ddej	dej	dB dej	dB de
e d	ef
d
dZe 		ddej	dej	dB dede
e d	eejB f
ddZ  ZS )ParakeetForCTCr'   c                    s<   t  | t|j| _tj|jj|jdd| _	| 
  d S )Nr   r   )r/   r0   r'  r  r(  r   rx   r3   
vocab_sizectc_headr4  rh   r:   r#   r$   r0     s   zParakeetForCTC.__init__Nr   r   labelsr   r   c              
   K   s  | j d||d|}|j}| |dddd}d}|dur|dur'|ntj|tjd}| |d}	|| j	j
k}
|
d}||
}tjj|dtjddd}tjjjd	d
 tjj|||	|| j	j
| j	j| j	jd}W d   n1 s{w   Y  t|||j|jdS )a  
        Example:

        ```python
        >>> from transformers import AutoProcessor, ParakeetForCTC
        >>> from datasets import load_dataset, Audio

        >>> model_id = "nvidia/parakeet-ctc-1.1b"
        >>> processor = AutoProcessor.from_pretrained(model_id)
        >>> model = ParakeetForCTC.from_pretrained(model_id)

        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate))

        >>> inputs = processor(ds[0]["audio"]["array"], text=ds[0]["text"])
        >>> outputs = model(**inputs)

        >>> print(outputs.loss)
        ```r   r   r   r*   Nr+   r>   r   r   F)rC   )blank	reductionzero_infinity)lossr?  r<   r  r#   )r(  r7  rD  rM   r    	ones_likelongr  r   r'   pad_token_idmasked_selectr   rl   log_softmaxr   backendscudnnflagsctc_lossctc_loss_reductionctc_zero_infinityr   r<   r  )r8   r   r   rE  r   encoder_outputsr<   r?  rJ  r   labels_masktarget_lengthsflattened_targets	log_probsr#   r#   r$   rX     sD   

zParakeetForCTC.forwardFreturn_dict_in_generatec                 K   st   d|d< | j d
||d|}|jjdd}|dur+| j||jd d}| jj|| < |r8t||j|j|j	d	S |S )a3  
        Example:

        ```python
        >>> from transformers import AutoProcessor, ParakeetForCTC
        >>> from datasets import load_dataset, Audio

        >>> model_id = "nvidia/parakeet-ctc-1.1b"
        >>> processor = AutoProcessor.from_pretrained(model_id)
        >>> model = ParakeetForCTC.from_pretrained(model_id)

        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate))

        >>> inputs = processor(ds[0]["audio"]["array"], text=ds[0]["text"])
        >>> predicted_ids = model.generate(**inputs)
        >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)

        >>> print(transcription)
        ```
        Treturn_dictrF  r>   rD   Nr   r6  )r>  r?  r  r<   r#   )
rX   r?  argmaxr  rG   r'   rM  r=  r  r<   )r8   r   r   r[  r   outputsr>  r#   r#   r$   generate  s&   zParakeetForCTC.generater  r   )r   r   r   r   r"   r0   r   r   r    r!   r   r   r   rX   r[   r   r=  r@  r_  r\   r#   r#   r:   r$   rB    s@   
 GrB  )rB  r'  r  )r   )r   ):r   collections.abcr   dataclassesr   r    r    r   r  activationsr   integrationsr   r	   modeling_layersr
   modeling_outputsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.genericr   r   utils.output_capturingr   configuration_parakeetr   r   r   Moduler%   r]   ro   r   r   r!   r   r   r6   r   r   r   r   r  r'  r=  rB  __all__r#   r#   r#   r$   <module>   s   2H
bE/F^ 