o ߥin&@sddlZddlZddlmZddlmmZGdddejZGdddejZ GdddejZ Gdd d ejZ Gd d d ejZ Gd d d ejZ GdddejZGdddejZGdddejZdddZdS)Nc(eZdZdZfddZddZZS)Q2VRankerStage1z Used to calculate the qv_ctx_score with query embedding and multi anchor context embeddings as input. The qv_ctx_score is used to pre-rank and retain top-k related anchors. c"tt|||_||_dSNsuper__init__nnLinearfcnscalesselfr hidden_dim __class___/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/multi_modal/soonet/blocks.pyr  zQ2VRankerStage1.__init__c CsX||}t}t|jD]}tdtj||dddtj|ddd}||q |SNz bld,bd->blpdim) r listranger torcheinsumF normalizeappend)r ctx_featsqfeat qv_ctx_scoresiscorerrrforwards  zQ2VRankerStage1.forward__name__ __module__ __qualname____doc__rr' __classcell__rrrrr s rcr)V2QRankerStage1zt Used to calculate the vq_ctx_score with anchor context embeddings and multi query embeddings as input. crrrr rrrr%rzV2QRankerStage1.__init__c CsTt}t|jD]}tdtj|||dddtj|ddd}||q|Sr) rrr rrrr r r!)rr"r# vq_ctx_scoresr%r&rrrr'*s zV2QRankerStage1.forwardr(rrrrr.  r.c*eZdZdZdfdd ZddZZS)Q2VRankerStage2z Used to calculate the qv_ctn_score with query embedding and video sequence embedding as input. The qv_ctn_score is used to re-rank anchors. cs0t||_||_t|||_t|_dSr) rrr snippet_lengthr r qfc V2VAttentionencoder)rr rr4rrrr;s  zQ2VRankerStage2.__init__c Cs.||}t}t}|\}}} t} t|jD]w} |jd| } || || dks0Jt|| d|| } ||| | |  }t|d|| }| |tj |dd|j d}| |tdtj|ddddtj|ddd}tj|dd\}}| || | |q||| fS) Nrrr)device bkld,bd->bklrr)r5rsizerr r4r index_selectviewdetachr7onesr8r!rrr unsqueezemax)rvfeatsr# hit_indicesr$ qv_ctn_scoresqv_merge_scores_LD ctn_featsr% anchor_length qv_ctx_scorectn_feat qv_ctn_scorerrrr'Bs<      zQ2VRankerStage2.forward)r3r(rrrrr25sr2cr)V2QRankerStage2zt Used to calculate the vq_ctn_score with anchor content embeddings and multi query embeddings as input. crrrr rrrrgrzV2QRankerStage2.__init__c Csht}t|jD])}tdtj|||ddddtj|ddd}tj |dd}| |q|S)Nr9rrr:rrr;) rrr rrrr r rAmeanr!)rrJr# vq_ctn_scoresr%r&rrrr'ls zV2QRankerStage2.forwardr(rrrrrObr0rOcr)r6z` Self-attention encoder for anchor frame sequence to encode intra-anchor knowledge. cs:ttdddd|_tdddd|_td|_dS)Nii)max_lenrdropoutg?)rn_headsrT) rrPositionEncodingposembMultiHeadAttentionr7r DropoutrT)rrrrr~s zV2VAttention.__init__cCsXtd||d}|}|||}|j||||d}||||d}|S)Nz bm,bn->bmnr)querykeyvaluemaskr)rrrArXr7rTfloat)r video_feats video_masksr^residualoutrrrr's  zV2VAttention.forwardr(rrrrr6y r6cr1) BboxRegressorzK Predict the offset of bounding box for each candidate anchor. Fcstt|||_t|||_|r6t|||_t||_t td||t t|d|_ nt t||t t|d|_ ||_ dS)Nr) rrr r fc_ctxfc_qfc_ctn SelfAttentionattn SequentialReLU predictor enable_stage2)rrrnrrrrs     zBboxRegressor.__init__c Cs||}tj|dd}t||t|d}|jr`|r`t}t t |D]$}t| ||dt|dd}| |}| |q*tj|dd}tj||gdd}n|}||}|S)Nrr;r)rgrcatrrelurfrArnrrlenrhrjr!rm) rr"rJr#ctx_fuse_featsctn_fuse_featsr%rc fuse_featsrrrr's$     zBboxRegressor.forward)Fr(rrrrresrecr)riz? Obtain pooled features by self-attentive pooling. cs<tt||d|_t|_t|dd|_dS)Nrr)rrr r fc1rlrqfc2)rrrrrrs  zSelfAttention.__init__cCsF||||d}tj|ddd}tj||dd}|S)Nr:rr;) rwrqrvsqueezersoftmaxrArsum)rxattrcrrrr'szSelfAttention.forwardr(rrrrrirdricr1)rWa! An implementation of trainable positional embedding which is added to sequence features to inject time/position information. Args: max_len: The max number of trainable positional embeddings. dim: the dimension of positional embedding. rRcs6tt|t|||_t|_t||_ dSr) rrWrr EmbeddingembedrlrqrZrT)rrSrrTrrrrs zPositionEncoding.__init__cCsR|jdd\}}tj|tj|jd}|d|d}||| |}|S)Nr)dtyper8rr) shaperarangelongr8rArepeatrTrqr~)rr{ batch_sizeseq_lenpos_idspos_embrrrr's zPositionEncoding.forwardrRr(rrrrrWs rWcs2eZdZdZd fdd ZddZddZZS) rYa  An implementation of multi-head attention module, as described in 'Attention Is All You Need ' Args: dim: the dimension of features of hidden layers. n_heads: the number of head. rRcsltt|||_||_|||_t|||_t|||_ t|||_ t ||_ tj dd|_dS)Nror;)rrYrrrVhead_dimr r to_qto_kto_vrZrTSoftmaxry)rrrVrTrrrrs  zMultiHeadAttention.__init__cCs6|dd|j|jf}|j|}|ddddS)Nrorrrr:)r<rVrr>permute)rr{ new_x_shaperrrtranspose_for_scoress z'MultiHeadAttention.transpose_for_scorescCs||}||}||}||}||} ||} t|| dd} | t|j } t | |} | | } | | } t| | } | dddd} | dd|jf} | j| } | S)Nrorrrr:)rrrrrmatmul transposemathsqrtr mask_logitsryrTr contiguousr<rr>)rr[r\r]r^qkvq_transk_transv_transr|ctx_vrrrrr's$            zMultiHeadAttention.forwardr)r)r*r+r,rrr'r-rrrrrYs  rYꌠ9Y>)cCs|tj}|d||S)Ng?)typerfloat32)inputsr^ mask_valuerrrrs r)r)rrtorch.nnr torch.nn.functional functionalrModulerr.r2rOr6rerirWrYrrrrrs --4