o ۷i@sdZddlZddlmZddlmZddlmZmZddl Z ddl m Z ddl m Z m Z mZdd lmZmZer?dd lmZdd lmZdd lmZdd lmZmZmZmZmZmZmZm Z m!Z!ddl"m#Z#ddl$m%Z%ddlm&Z&ddl'm(Z(e&)e*Z+ddZ,ddZ-ddZ.Gddde j/Z0Gddde j/Z1Gddde j/Z2Gdd d e j/Z3Gd!d"d"e j/Z4Gd#d$d$e j/Z5Gd%d&d&eZ6Gd'd(d(e j/Z7Gd)d*d*e j/Z8Gd+d,d,e j/Z9Gd-d.d.e j/Z:Gd/d0d0e j/Z;Gd1d2d2e j/Zeed7d8Gd9d:d:eZ?eGd;d<dZ@ed=d8Gd>d?d?e>ZAeGd@dAdAe>ZBedBd8GdCdDdDe>ZCedEd8GdFdGdGe>ZDeGdHdIdIe>ZEeGdJdKdKe>ZFeGdLdMdMe>ZGgdNZHdS)OzPyTorch FNet model.N) dataclass)partial)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)auto_docstringis_scipy_available)linalg)ACT2FN)GradientCheckpointingLayer) BaseModelOutputBaseModelOutputWithPoolingMaskedLMOutput ModelOutputMultipleChoiceModelOutputNextSentencePredictorOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward)logging) FNetConfigcCs:|jd}|d|d|f}|tj}td|||S)z4Applies 2D matrix multiplication to 3D input arrays.rNzbij,jk,ni->bnk)shapetypetorch complex64einsum)xmatrix_dim_onematrix_dim_two seq_lengthr'\/home/ubuntu/vllm_env/lib/python3.10/site-packages/transformers/models/fnet/modeling_fnet.py_two_dim_matmul7s  r)cCs t|||SN)r))r#r$r%r'r'r(two_dim_matmul@ r+cCs4|}tt|jddD] }tjj||d}q |S)z Applies n-dimensional Fast Fourier Transform (FFT) to input array. Args: x: Input n-dimensional array. Returns: n-dimensional Fourier transform of input n-dimensional array. rN)axis)reversedrangendimr fft)r#outr-r'r'r(fftnEs r3cs*eZdZdZfddZdddZZS)FNetEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.csttj|j|j|jd|_t|j|j|_ t|j |j|_ tj |j|j d|_ t|j|j|_t|j|_|jdt|jddd|jdtj|jtjddddS) N) padding_idxeps position_ids)rF) persistenttoken_type_idsdtype)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsLinear projectionDropouthidden_dropout_probdropoutregister_bufferr arangeexpandzerosr8sizelongselfconfig __class__r'r(r?Xs  zFNetEmbeddings.__init__Nc Cs|dur |}n|dd}|d}|dur$|jddd|f}|durNt|drC|jddd|f}||d|}|}n tj|tj|jjd}|durW| |}| |} || } | |} | | 7} | | } | | } || } | S)Nr9rr;rr=device)rTr8hasattrr;rRr rSrUr\rDrHrFrIrLrO) rW input_idsr;r8 inputs_embeds input_shaper&buffered_token_type_ids buffered_token_type_ids_expandedrH embeddingsrFr'r'r(forwardns,        zFNetEmbeddings.forward)NNNN)__name__ __module__ __qualname____doc__r?rd __classcell__r'r'rYr(r4Us r4c,eZdZfddZddZddZZS)FNetBasicFourierTransformcst||dSr*)r>r?_init_fourier_transformrVrYr'r(r? z"FNetBasicFourierTransform.__init__cCs|jsttjjdd|_dS|jdkrLtrB|dtj t |j tj d|dtj t |jtj dtt|j|jd|_dStdt|_dSt|_dS) N)rdimdft_mat_hiddenr< dft_mat_seq)r$r%zpSciPy is needed for DFT matrix calculation and is not found. Using TPU optimized fast fourier transform instead.)use_tpu_fourier_optimizationsrr r1r3fourier_transformrEr rPtensorr dftrBr!tpu_short_seq_lengthr+rsrrrwarningrVr'r'r(rls$     z1FNetBasicFourierTransform._init_fourier_transformcCs||j}|fSr*)rureal)rW hidden_statesoutputsr'r'r(rds z!FNetBasicFourierTransform.forward)rerfrgr?rlrdrir'r'rYr(rks rkc$eZdZfddZddZZS)FNetBasicOutputcs"ttj|j|jd|_dSNr6)r>r?rrIrBrJrVrYr'r(r?s zFNetBasicOutput.__init__cCs|||}|Sr*)rIrWr{ input_tensorr'r'r(rdszFNetBasicOutput.forwardrerfrgr?rdrir'r'rYr(r~ r~cr})FNetFourierTransformcs"tt||_t||_dSr*)r>r?rkrWr~outputrVrYr'r(r?s  zFNetFourierTransform.__init__cCs$||}||d|}|f}|SNr)rWr)rWr{ self_outputsfourier_outputr|r'r'r(rds zFNetFourierTransform.forwardrr'r'rYr(r rc2eZdZfddZdejdejfddZZS)FNetIntermediatecsDtt|j|j|_t|jt rt |j|_ dS|j|_ dSr*) r>r?rrKrBintermediate_sizedense isinstance hidden_actstrrintermediate_act_fnrVrYr'r(r?s   zFNetIntermediate.__init__r{returncC||}||}|Sr*)rrrWr{r'r'r(rd  zFNetIntermediate.forwardrerfrgr?r Tensorrdrir'r'rYr(rs rcs8eZdZfddZdejdejdejfddZZS) FNetOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSr) r>r?rrKrrBrrIrJrMrNrOrVrYr'r(r?s zFNetOutput.__init__r{rrcCs&||}||}|||}|Sr*)rrOrIrr'r'r(rds  zFNetOutput.forwardrr'r'rYr(rs $rcrj) FNetLayercs:t|j|_d|_t||_t||_t||_ dSNr) r>r?chunk_size_feed_forward seq_len_dimrfourierr intermediaterrrVrYr'r(r?s   zFNetLayer.__init__cCs0||}|d}t|j|j|j|}|f}|Sr)rrfeed_forward_chunkrr)rWr{self_fourier_outputsr layer_outputr|r'r'r(rds zFNetLayer.forwardcCs||}|||}|Sr*)rr)rWrintermediate_outputrr'r'r(rs  zFNetLayer.feed_forward_chunk)rerfrgr?rdrrir'r'rYr(rs  rcs&eZdZfddZdddZZS) FNetEncodercs:t|_tfddtjD|_d|_dS)Ncsg|]}tqSr')r).0_rXr'r( sz(FNetEncoder.__init__..F) r>r?rXr ModuleListr/num_hidden_layerslayergradient_checkpointingrVrYrr(r? s   zFNetEncoder.__init__FTcCsr|rdnd}t|jD]\}}|r||f}||}|d}q |r&||f}|s3tdd||fDSt||dS)Nr'rcss|] }|dur|VqdSr*r')rvr'r'r( sz&FNetEncoder.forward..)last_hidden_stater{) enumeratertupler)rWr{output_hidden_states return_dictall_hidden_statesi layer_module layer_outputsr'r'r(rds     zFNetEncoder.forward)FTrr'r'rYr(r s rcr) FNetPoolercs*tt|j|j|_t|_dSr*)r>r?rrKrBrTanh activationrVrYr'r(r?&s zFNetPooler.__init__r{rcCs(|dddf}||}||}|Sr)rr)rWr{first_token_tensor pooled_outputr'r'r(rd+s  zFNetPooler.forwardrr'r'rYr(r%s rcr)FNetPredictionHeadTransformcsVtt|j|j|_t|jtrt |j|_ n|j|_ tj |j|j d|_ dSr) r>r?rrKrBrrrrrtransform_act_fnrIrJrVrYr'r(r?6s  z$FNetPredictionHeadTransform.__init__r{rcCs"||}||}||}|Sr*)rrrIrr'r'r(rd?s   z#FNetPredictionHeadTransform.forwardrr'r'rYr(r5s  rcs.eZdZfddZddZd ddZZS) FNetLMPredictionHeadcsHtt||_t|j|j|_t t |j|_ |j |j_ dSr*) r>r?r transformrrKrBrAdecoder Parameterr rSbiasrVrYr'r(r?Gs  zFNetLMPredictionHead.__init__cCrr*)rrrr'r'r(rdRrzFNetLMPredictionHead.forwardrNcCs,|jjjjdkr|j|j_dS|jj|_dS)Nmeta)rrr\rrWr'r'r( _tie_weightsWsz!FNetLMPredictionHead._tie_weights)rN)rerfrgr?rdrrir'r'rYr(rFs  rcr})FNetOnlyMLMHeadcstt||_dSr*)r>r?r predictionsrVrYr'r(r?armzFNetOnlyMLMHead.__init__cC||}|Sr*)r)rWsequence_outputprediction_scoresr'r'r(rde zFNetOnlyMLMHead.forwardrr'r'rYr(r`rrcr})FNetOnlyNSPHeadcstt|jd|_dSNrn)r>r?rrKrBseq_relationshiprVrYr'r(r?ls zFNetOnlyNSPHead.__init__cCrr*)r)rWrseq_relationship_scorer'r'r(rdprzFNetOnlyNSPHead.forwardrr'r'rYr(rkrrcr})FNetPreTrainingHeadscs(tt||_t|jd|_dSr)r>r?rrrrKrBrrVrYr'r(r?ws  zFNetPreTrainingHeads.__init__cCs||}||}||fSr*)rr)rWrrrrr'r'r(rd|s  zFNetPreTrainingHeads.forwardrr'r'rYr(rvrrc@s&eZdZUeed<dZdZddZdS)FNetPreTrainedModelrXfnetTcCst|tjr |jjjd|jjd|jdur|jj dSdSt|tj rC|jjjd|jjd|j durA|jj|j  dSdSt|tj rX|jj |jj ddSdS)zInitialize the weightsg)meanstdNg?)rrrKweightdatanormal_rXinitializer_rangerzero_r@r5rIfill_)rWmoduler'r'r( _init_weightss      z!FNetPreTrainedModel._init_weightsN)rerfrgr__annotations__base_model_prefixsupports_gradient_checkpointingrr'r'r'r(rs  rz0 Output type of [`FNetForPreTraining`]. ) custom_introc@s^eZdZUdZdZeejed<dZ eejed<dZ eejed<dZ ee ejed<dS)FNetForPreTrainingOutputa loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`): Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). Nlossprediction_logitsseq_relationship_logitsr{) rerfrgrhrrr FloatTensorrrrr{rr'r'r'r(rs  rcseZdZdZdfdd ZddZddZe dd ee j d ee j d ee j d ee j dee dee de eeffddZZS) FNetModelz The model can behave as an encoder, following the architecture described in [FNet: Mixing Tokens with Fourier Transforms](https://huggingface.co/papers/2105.03824) by James Lee-Thorp, Joshua Ainslie, Ilya Eckstein, Santiago Ontanon. TcsDt|||_t||_t||_|rt|nd|_| dS)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer N) r>r?rXr4rcrencoderrpooler post_init)rWrXadd_pooling_layerrYr'r(r?s    zFNetModel.__init__cCs|jjSr*rcrDrr'r'r(get_input_embeddingsszFNetModel.get_input_embeddingscCs ||j_dSr*r)rWvaluer'r'r(set_input_embeddingsr,zFNetModel.set_input_embeddingsNr^r;r8r_rrrcCsv|dur|n|jj}|dur|n|jj}|dur |dur td|dur-|}|\}} n|dur>|dd}|\}} ntd|jjrT| dkrT|jj| krTtd|dur[|jn|j} |durt|j dr}|j j ddd| f} | || } | }n t j |t j| d}|j ||||d} |j| ||d }|d }|jdur||nd}|s||f|d dSt|||jd S) NzDYou cannot specify both input_ids and inputs_embeds at the same timer9z5You have to specify either input_ids or inputs_embedsrqzThe `tpu_short_seq_length` in FNetConfig should be set equal to the sequence length being passed to the model when using TPU optimizations.r;r[)r^r8r;r_)rrrr)r pooler_outputr{)rXruse_return_dict ValueErrorrTrtrxr\r]rcr;rRr rSrUrrrr{)rWr^r;r8r_rrr` batch_sizer&r\rarbembedding_outputencoder_outputsrrr'r'r(rds\      zFNetModel.forward)T)NNNNNN)rerfrgrhr?rrr rr LongTensorrboolrrrrdrir'r'rYr(rs6 rz FNet Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. cseZdZddgZfddZddZddZe dd ee j d ee j d ee j d ee j dee j dee j dee dee de e effddZZS)FNetForPreTrainingcls.predictions.decoder.biascls.predictions.decoder.weightc,t|t||_t||_|dSr*)r>r?rrrclsrrVrYr'r(r?!    zFNetForPreTraining.__init__cC |jjjSr*rrrrr'r'r(get_output_embeddings* z(FNetForPreTraining.get_output_embeddingscC||jj_|j|jj_dSr*rrrrrWnew_embeddingsr'r'r(set_output_embeddings- z(FNetForPreTraining.set_output_embeddingsNr^r;r8r_labelsnext_sentence_labelrrrc Cs|dur|n|jj}|j||||||d} | dd\} } || | \} } d}|durP|durPt}|| d|jj|d}|| dd|d}||}|sg| | f| dd}|dure|f|S|St|| | | jdS)aH labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, FNetForPreTraining >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/fnet-base") >>> model = FNetForPreTraining.from_pretrained("google/fnet-base") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.prediction_logits >>> seq_relationship_logits = outputs.seq_relationship_logits ```Nr;r8r_rrrnr9)rrrr{) rXrrrrviewrArr{)rWr^r;r8r_rrrrr|rrrr total_lossloss_fctmasked_lm_lossnext_sentence_lossrr'r'r(rd1s4% zFNetForPreTraining.forwardNNNNNNNN)rerfrg_tied_weights_keysr?rrr rr rrrrrrdrir'r'rYr(rsB    rcseZdZddgZfddZddZddZe dd ee j d ee j d ee j d ee j dee j dee dee de e effddZZS)FNetForMaskedLMrrcrr*)r>r?rrrrrrVrYr'r(r?{rzFNetForMaskedLM.__init__cCrr*rrr'r'r(rrz%FNetForMaskedLM.get_output_embeddingscCrr*rrr'r'r(rrz%FNetForMaskedLM.set_output_embeddingsNr^r;r8r_rrrrcCs|dur|n|jj}|j||||||d}|d} || } d} |dur5t} | | d|jj|d} |sK| f|dd} | durI| f| S| St| | |jdS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Nr rr9rnrlogitsr{) rXrrrrr rArr{)rWr^r;r8r_rrrr|rrr r rr'r'r(rds& zFNetForMaskedLM.forwardNNNNNNN)rerfrgrr?rrr rr rrrrrrdrir'r'rYr(rws<   rzT FNet Model with a `next sentence prediction (classification)` head on top. ceZdZfddZe       ddeejdeejdeejdeejdeejd eed eed e e e ffd d Z Z S)FNetForNextSentencePredictioncrr*)r>r?rrrrrrVrYr'r(r?rz&FNetForNextSentencePrediction.__init__Nr^r;r8r_rrrrcKsd|vrtdt|d}|dur|n|jj}|j||||||d} | d} || } d} |durBt} | | dd| d} |sX| f| dd}| durV| f|S|St | | | j dS) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring). Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, FNetForNextSentencePrediction >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/fnet-base") >>> model = FNetForNextSentencePrediction.from_pretrained("google/fnet-base") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt") >>> outputs = model(**encoding, labels=torch.LongTensor([1])) >>> logits = outputs.logits >>> assert logits[0, 0] < logits[0, 1] # next sentence was random ```rzoThe `next_sentence_label` argument is deprecated and will be removed in a future version, use `labels` instead.Nr rr9rnr) warningswarn FutureWarningpoprXrrrrr rr{)rWr^r;r8r_rrrkwargsr|rseq_relationship_scoresrr rr'r'r(rds:$  z%FNetForNextSentencePrediction.forwardr)rerfrgr?r rr rrrrrrdrir'r'rYr(rs6   rz FNet Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cr)FNetForSequenceClassificationcJt||j|_t||_t|j|_t |j |j|_ | dSr* r>r? num_labelsrrrrMrNrOrKrB classifierrrVrYr'r(r?s   z&FNetForSequenceClassification.__init__Nr^r;r8r_rrrrcCsh|dur|n|jj}|j||||||d}|d} || } || } d} |dur|jjdurS|jdkr9d|j_n|jdkrO|jtj ksJ|jtj krOd|j_nd|j_|jjdkrqt } |jdkrk| | | } n+| | |} n%|jjdkrt } | | d|j|d} n|jjdkrt} | | |} |s| f|dd} | dur| f| S| St| | |jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr r regressionsingle_label_classificationmulti_label_classificationr9rnr)rXrrrOr! problem_typer r=r rUintr squeezerr rrr{)rWr^r;r8r_rrrr|rrrr rr'r'r(rd"sF     "       z%FNetForSequenceClassification.forwardr)rerfrgr?r rr rrrrrrdrir'r'rYr(rs6   rcr)FNetForMultipleChoicecs@t|t||_t|j|_t|j d|_ | dSr) r>r?rrrrMrNrOrKrBr!rrVrYr'r(r?as   zFNetForMultipleChoice.__init__Nr^r;r8r_rrrrcCsF|dur|n|jj}|dur|jdn|jd}|dur%|d|dnd}|dur4|d|dnd}|durC|d|dnd}|durV|d|d|dnd}|j||||||d} | d} || } || } | d|} d} |durt}|| |} |s| f| dd}| dur| f|S|St | | | j dS)a[ input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nrr9r rnr) rXrrr rTrrOr!rrr{)rWr^r;r8r_rrr num_choicesr|rrreshaped_logitsrr rr'r'r(rdks:)    zFNetForMultipleChoice.forwardr)rerfrgr?r rr rrrrrrdrir'r'rYr(r(_s6   r(cr)FNetForTokenClassificationcrr*rrVrYr'r(r?s   z#FNetForTokenClassification.__init__Nr^r;r8r_rrrrcCs|dur|n|jj}|j||||||d}|d} || } || } d} |dur9t} | | d|j|d} |sO| f|dd} | durM| f| S| St| | |j dS)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Nr rr9rnr) rXrrrOr!rr r rr{)rWr^r;r8r_rrrr|rrrr rr'r'r(rds(  z"FNetForTokenClassification.forwardr)rerfrgr?r rr rrrrrrdrir'r'rYr(r,s6   r,cseZdZfddZe        ddeejdeejdeejdeejdeejd eejd eed eed e e e ffd dZ Z S)FNetForQuestionAnsweringcs<t||j|_t||_t|j|j|_| dSr*) r>r?r rrrrKrB qa_outputsrrVrYr'r(r?s   z!FNetForQuestionAnswering.__init__Nr^r;r8r_start_positions end_positionsrrrc Cs>|dur|n|jj}|j||||||d} | d} || } | jddd\} } | d} | d} d}|dur|durt|dkrL|d}t|dkrY|d}| d}| d|}| d|}t |d}|| |}|| |}||d}|s| | f| dd}|dur|f|S|St || | | j dS) Nr rrr9ro) ignore_indexrn)r start_logits end_logitsr{) rXrrr.splitr' contiguouslenrTclamprrr{)rWr^r;r8r_r/r0rrr|rrr2r3r  ignored_indexr  start_lossend_lossrr'r'r(rdsB            z FNetForQuestionAnswering.forwardr)rerfrgr?r rr rrrrrrdrir'r'rYr(r-s<    r-) rr(rrr-rr,rrr)Irhr dataclassesr functoolsrtypingrrr rtorch.nnrrr utilsr r scipyr activationsrmodeling_layersrmodeling_outputsrrrrrrrrrmodeling_utilsr pytorch_utilsrrconfiguration_fnetr get_loggerreloggerr)r+r3Moduler4rkr~rrrrrrrrrrrrrrrrrrr(r,r-__all__r'r'r'r(s      ,     =&    eY>UI[9 D