o ϯi&@sddlmZddlmZmZmZddlZddlmZddl m m Z ddl m Z mZGdddejZGdddejZGd d d ejZGd d d eZGd ddejZeejddZGdddejZdS))partial)CallableListOptionalN)DropPath trunc_normal_cs0eZdZ     d fdd ZddZZS) AttentionFNcsft||_||}|p|d|_tj||d|d|_t||_t|||_ t||_ dS)Ng)bias) super__init__ num_headsscalennLinearqkvDropout attn_dropproj proj_drop)selfdimrqkv_biasqk_scalerrhead_dim __class__P/home/ubuntu/.local/lib/python3.10/site-packages/imagebind/models/transformer.pyrs  zAttention.__init__c Cs|j\}}}||||d|j||jddddd}|d|d|d}}}||dd|j} | jdd} || } | |dd|||}| |}| |}|S) Nr r)r) shaperreshaperpermute transposersoftmaxrrr) rxBNCrqkvattnrrr forward-s      zAttention.forward)r FNr r )__name__ __module__ __qualname__rr3 __classcell__rrrr rsrcs0eZdZddejdffdd ZddZZS)MlpNr csNt|p|}|p |}t|||_||_t|||_t||_dSN) r rrrfc1actfc2rdrop)r in_featureshidden_features out_features act_layerr=rrr rEs z Mlp.__init__cCs6||}||}||}||}||}|Sr9)r:r;r=r<)rr+rrr r3Us     z Mlp.forward)r4r5r6rGELUrr3r7rrrr r8Dsr8c*eZdZdejdejffdd ZZS)MultiheadAttentionr+ attn_maskcstj|||d|ddS)NF) need_weightsrErr r3rr+rErrr r3_szMultiheadAttention.forwardr4r5r6torchTensorr3r7rrrr rD^"rDcrC) ViTAttentionr+rEcs|dusJt|Sr9rGrHrrr r3ds  zViTAttention.forwardrIrrrr rMcrLrMcspeZdZdejejddddfdedededed ed ed ed e e d effdd Z de j de j fddZZS)BlockWithMaskingr#r N-C6?r attn_target mlp_ratiorA norm_layerffn_dropout_rate drop_pathlayer_scale_typelayer_scale_init_valuec stt|tjrJd||_|dkrt||_nt|_|||_ t ||} t || ||d|_ |||_ ||_|jdur|jdvsPJd|j|jdkr[dd|g} n |jdkrdgd } tjtj| d | d d |_tjtj| d | d d |_dSdS) NzPattn_target should be a Callable. Otherwise attn_target is shared across blocks!r )r>r?rAr=) per_channelscalarzFound Layer scale type rWr"rX)r"r"r")sizeT) requires_grad)r r isinstancerModuler2rrTIdentitynorm_1intr8mlpnorm_2rU ParameterrJoneslayer_scale_gamma1layer_scale_gamma2) rrrPrQrArRrSrTrUrVmlp_hidden_dim gamma_shaperrr rjsJ             zBlockWithMasking.__init__r+rEcCs|jdur"||||||}|||||}|S|||||||j}||||||j}|Sr9)rUrTr2r^r`rardrerHrrr r3s  zBlockWithMasking.forward)r4r5r6rrB LayerNormr_rfloatrstrrrJrKr3r7rrrr rNis8  5rNgư>)epscseZdZeddddedddddf deded ed ed eed eed ede dedededee dede ffdd Z ddZ    d"de j de j dededeeef d d!ZZS)#SimpleTransformerNr progressiver#rOjaxrP embed_dim num_blocksblockpre_transformer_layerpost_transformer_layerdrop_path_ratedrop_path_typerRrQrSrUrVweight_init_stylec st||_|dkrddtd|Dn|dkr)fddt|Dntd|tj f ddt|D|_ ||_ ||_ | |j d S) a Simple Transformer with the following features 1. Supports masked attention 2. Supports DropPath 3. Supports LayerScale 4. Supports Dropout in Attention and FFN 5. Makes few assumptions about the input except that it is a Tensor rmcSsg|]}|qSr)item).0r+rrr sz.SimpleTransformer.__init__..runiformcsg|]}qSrrrxi)rtrr ryszUnknown drop_path_type: c s(g|]}|dqS))rrPrQrSrTrRrUrVrr{) rPrqdprrorSrVrUrQrRrr rys N)r rrrrJlinspacerange ValueErrorr Sequentialblocksrsrvapply _init_weights)rrProrprqrrrsrtrurRrQrSrUrVrvr) rPrqr}rtrorSrVrUrQrRr rs  zSimpleTransformer.__init__cCst|tjr1|jdkrtjj|jn |jdkr t|jdd|j dur/tj |j ddSdSt|tj rItj |j dtj |jddSdS)Nrnpytorchg{Gz?)stdrg?) r[rrrvrJinitxavier_uniform_weightrr constant_rh)rmrrr rs     zSimpleTransformer._init_weightsFr"tokensrEuse_checkpointcheckpoint_every_ncheckpoint_blk_idscs|jr||}|r|durfddtt|jD}|r"t|}t|jD]\}}|r;||vr;tj|||dd}q'|||d}q'|jrJ||}|S)a  Inputs - tokens: data of shape N x L x D (or L x N x D depending on the attention implementation) - attn: mask of shape L x L Output - x: data of shape N x L x D (or L x N x D depending on the attention implementation) Ncsg|] }|dkr|qS)rr)rxblk_idrrr rys  z-SimpleTransformer.forward..F) use_reentrant)rE)rrrlenrset enumerate checkpointrs)rrrErrrrblkrrr r3s"      zSimpleTransformer.forward)NFr"N)r4r5r6rN _LAYER_NORMrr_rrirjrrrJrKboolrr3r7rrrr rlsr     5 rl) functoolsrtypingrrrrJtorch.nnrtorch.utils.checkpointutilsr timm.layersrrr\rr8rDrMrNrhrrlrrrr s  -D