o ϯimZ@sddlZddlZddlZddlZddlmZddlmZmZm Z m Z ddl Z ddl Z ddlZddlZddlmZddlmZddlmZddlmZmZddZd d Z  d.d d Z d/ddZGdddejZGdddeZ GdddeZ!Gddde!Z"Gddde!Z#ddZ$GdddeZ%GdddejZ&Gd d!d!e&Z'ed"d#Z(d$d%Z)d&d'Z*d(d)Z+Gd*d+d+e,Z-Gd,d-d-eZ.dS)0N) lru_cache)CallableListOptionalTuple) g_pathmgr) trunc_normal_)VerboseNNModulecast_if_src_dtypecsfddtfddt|D}t|dddddf|dddddf<t|dddddf|dddddf<t|dS) z Sinusoid position encoding tablecsfddtDS)Nc s(g|]}tdd|dqS)i')nppower).0hid_j)d_hidposition]/home/ubuntu/.local/lib/python3.10/site-packages/imagebind/models/multimodal_preprocessors.py szOget_sinusoid_encoding_table..get_position_angle_vec..)ranger)rrrget_position_angle_vecs z;get_sinusoid_encoding_table..get_position_angle_veccsg|]}|qSrr)rpos_i)rrrr%z/get_sinusoid_encoding_table..Nrr )r arrayrsincostorch FloatTensor unsqueeze) n_positionrsinusoid_tabler)rrrget_sinusoid_encoding_tables ..r#c Cs|jd}||kr |S|jd}t|tjtj\}}tjj|dt t |t t || ddddt ||dd}|rJt|tjtj\}}| dddd dd|}|S)Nrrr bicubic) scale_factormode)shaper rbfloat16float32nn functional interpolatereshapeintmathsqrtpermuteview)target_spatial_size pos_embedNdimupdated_rrrinterpolate_pos_encoding_2d-s  $ r;rc Cs |dks |dks Jd|jd|}||kr|S|d|dks%Jd|ddd|f}|dd|df}|dusC|ddkrIt||}n3|ddkrxt|dksYJd|d}|d|d }|d||d}t||d d}ntd tj||fdd S) Nrrzthere is 1 CLS token or noner$z?Interpolation of pos embed not supported for non-square layoutsz$temporal interpolation not supportedr )rr.z,This type of interpolation isn't implementedr8)r)r;lenr4r ValueErrorrcat) npatch_per_imgr6patches_layout input_shapefirst_patch_idxr7 class_emb num_framesnum_spatial_tokensrrrinterpolate_pos_encodingAs*  rIcCst|||||d}|S)N)rDrE)rI)rBr6rCrDrErrr_get_pos_embeddinghsrJcs>eZdZdZd deejffdd ZddZdd Z Z S) PatchEmbedGenericz PatchEmbed from Hydra N norm_layercs8tt|dkrtj||_n|d|_||_dSNrr)super__init__r?r, SequentialprojrL)self proj_stemrL __class__rrrO~s    zPatchEmbedGeneric.__init__cCsrttdg|}||}Wdn1swY|jd}t|jdd}t|}|||fS)Nrr )rno_gradzerosrQr)tupler prod)rRimg_size dummy_img dummy_out embed_dimrC num_patchesrrrget_patch_layouts     z"PatchEmbedGeneric.get_patch_layoutcCs4||}|ddd}|jdur||}|S)Nr r)rQflatten transposerLrRxrrrforwards   zPatchEmbedGeneric.forwardN) __name__ __module__ __qualname____doc__rr,ModulerOr_rd __classcell__rrrTrrKys  rKc s>eZdZdedededededdf fdd Zd d ZZS) SpatioTemporalPosEmbeddingHelperrCr^num_cls_tokensr] learnablereturnNcstt||_||_||_|||_||_|jr.tt d|j||_ t |j dddS| dt|j|dS)Nr{Gz?stdr6)rNrOrmrCr^ num_tokensrnr, ParameterrrWr6rregister_bufferr#)rRrCr^rmr]rnrTrrrOs   z)SpatioTemporalPosEmbeddingHelper.__init__cCs.|j}t|d|j|j|j||jd}|S)Nr)r6rCrDrE)r)rJsizermr6rC)rR vision_inputall_vision_tokensrDr6rrrget_pos_embeddingsz2SpatioTemporalPosEmbeddingHelper.get_pos_embedding) rfrgrhrr0boolrOryrkrrrTrrlsrlcsteZdZ     ddedeeded ed eed ed e d dffdd Z e ddZ ddZdddZZS)RGBDTPreprocessorr%r}rNFopenclip rgbt_stem depth_stemrZrm pos_embed_fnuse_type_embedinit_param_styleroc st|dur |n|}||\|_|_|_||_||_|du|_||_ ||_ |jr8||j||j|jd|_ |j dkrJt td|j |j|_|j rYt tdd|j|_||dS)N)rCrmr^r]rr)rNrOr_rCr^r]rr use_pos_embedrrmpos_embedding_helperr,rtrrW cls_token type_embedinit_parameters) rRrrrZrmrrrstemrTrrrOs4   zRGBDTPreprocessor.__init__cCs|dkr0|jd}|jrtj|jj|jj|9_|jdkr/tj|j|j|9_n|dkr<|jj dnt d||j rOtj|j dSdS)Nr~rvit Unknown init )r]rr,initnormal_rr6rmrdatafill_r@rrrRrscalerrrrs  z!RGBDTPreprocessor.init_parameterscCs||}|jdks J|jd|jksJ|jd}|jdkr0|j|dd}tj||fdd}|jr>|j ||}||}|j rK||j |dd}|SNr%r rr$rr>) ndimr)r]rmrexpandrrArrryrr)rRinputrmasktokensB class_tokensr6rrrtokenize_input_and_cls_poss  z,RGBDTPreprocessor.tokenize_input_and_cls_poscCsz|durt|dur|||j|}|dur|||j|}|dur,|dur,||}n|dur2|n|}d|iid}|S)Nrtrunkhead)NotImplementedErrorrrr)rRvisiondepth patch_mask vision_tokens depth_tokens final_tokens return_dictrrrrds$ zRGBDTPreprocessor.forward)r|rNFr~)NNN)rfrgrhrKrrr0rrzstrrOrrVrrrdrkrrrTrr{s6 ' r{c4eZdZdeddffdd Zdfdd ZZS) AudioPreprocessor audio_stemroNc tjd|dd|dSN)rrrrNrO)rRrkwargsrTrrrO.zAudioPreprocessor.__init__ctj|dSN)rrNrd)rRaudiorTrrrd1zAudioPreprocessor.forwardrerfrgrhrKrOrdrkrrrTrr-rcr) ThermalPreprocessor thermal_stemroNc rrr)rRrrrTrrrO6rzThermalPreprocessor.__init__crrr)rRthermalrTrrrd9rzThermalPreprocessor.forwardrerrrrTrr5rrcCs,tj||dd}|td|d|S)NF) requires_gradz-infr)remptyrfloattriu_)context_lengthrrrrbuild_causal_attention_mask=s rcs`eZdZ   ddededededed ed ed d ffd d ZedddZ ddZ Z S)TextPreprocessorTrr~ vocab_sizerr]causal_maskingsupply_seq_len_to_headrmrroNc st||_||_t|||_tt d|j|||_ ||_ |j r1t |j}| d|||_||_||_|dkrS|j dusGJdttd|j||_||dS)NrrrFz%Masking + CLS token isn't implemented)rNrOrrr, Embeddingtoken_embeddingrtrrr6rrrurrmr]rWrr) rRrrr]rrrmrrrTrrrOGs(   zTextPreprocessor.__init__cCstjj|jjddtjj|jdd|dkr3|jd}|jdkr1tj|j|j|9_dSdS|dkr@|jj ddSt d|) Nrprq{Gz?r~rrrr) r,rrrweightr6r]rmrrrr@rrrrris  z TextPreprocessor.init_parameterscCs||}|jdkr |jd}|j|dd}tj||fdd}||j}d|iid}|jr;|j dd}d|i|d<|j rH|d  d |j i|S) Nrr$rr>rrseq_lenrr attn_mask) rrmr)rrrrAr6rargmaxrupdater)rRtext text_tokensrrr text_lengthsrrrrdzs&     zTextPreprocessor.forward)Trr~)r~) rfrgrhr0rzrrOrrVrrdrkrrrTrrFs0 " rcs*eZdZdZdfdd ZddZZS)Im2Videoz&Convert an image into a trivial video.r cst||_dSre)rNrOtime_dim)rRrrTrrrOs  zIm2Video.__init__cCs4|jdkr ||jS|jdkr|Std|j)Nr=zDimension incorrect )rr rr@r)rbrrrrds   zIm2Video.forwardr )rfrgrhrirOrdrkrrrTrrsrcs*eZdZdfdd ZfddZZS) PadIm2Videor cs6tj|d|dks J|dvsJ||_||_dS)N)rr)zerorepeat)rNrOntimespad_type)rRrrrrTrrrOs    zPadIm2Video.__init__cst|}|j|jdkrM|jdkr(dgt|j}|j||j<||}|S|jdkrMddgt|j}|j|j|j|d|jd<tj ||}|S)Nrrrrr ) rNrdr)rrr?rrr,r-pad)rRrc new_shapepadargrTrrrds      zPadIm2Video.forwardr)rfrgrhrOrdrkrrrTrrsrcCstttdtddtttdtddtttdtdd}|dd}d }td D]}||vrI|||d ||d7}q3d d |D}tt||S) a9 Returns list of utf-8 byte and a corresponding list of unicode strings. The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. This is a signficant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup tables between utf-8 bytes and unicode strings. And avoids mapping to whitespace/control characters the bpe code barfs on. !~r¡¬®ÿNrcSsg|]}t|qSr)chr)rnrrrrrz$bytes_to_unicode..)listrordappenddictzip)bscsrbrrrbytes_to_unicodes     rcCs6t}|d}|ddD] }|||f|}q |S)zReturn set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings). rrN)setadd)wordpairs prev_charcharrrr get_pairss rcCs"t|}tt|}|Sre)ftfyfix_texthtmlunescapestriprrrr basic_cleans rcCstdd|}|}|S)Nz\s+ )resubrrrrrwhitespace_cleansrc@s>eZdZddefddZddZddZd d Zdd d Zd S)SimpleTokenizerMbpe_pathcCs@t|_dd|jD|_t|d}t|}t | d d}Wdn1s4wY|dd}dd |D}t t }|d d |D}|D] }|d |qX|d d gtt|tt||_dd|jD|_tt|tt||_d d d|_tdtj|_||_dS)NcSi|]\}}||qSrrrkvrrr z,SimpleTokenizer.__init__..rbutf-8 ricSsg|]}t|qSr)rXsplit)rmergerrrrsz,SimpleTokenizer.__init__..cSsg|]}|dqS)r)rrrrrrr<|startoftext|> <|endoftext|>cSrrrrrrrrr)rrz[<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+)r byte_encoderitems byte_decoderropenioBytesIOreadgzipdecoder rvaluesrjoinextendrrrr?encoderdecoder bpe_rankscachercompile IGNORECASEpatr)rRrrfh bpe_bytesmergesvocabr rrrrOs0  zSimpleTokenizer.__init__c sj|jvr j|St|dd|ddf}t|}|s#|dS t|fddd}|jvr4nu|\}}g}d}|t|krz|||} |||| | }Wn|||dYn3|||kr|t|dkr||d|kr||||d 7}n ||||d7}|t|ksBt|}|}t|dkrnt|}q$d |}|j|<|S) Nr$r Tcsj|tdS)Ninf)rgetr)pairrRrrsz%SimpleTokenizer.bpe..)keyrrr r) r rXrminrr?indexrrr) rRtokenrrbigramfirstsecondnew_wordijrr+rbpe sH     ,     zSimpleTokenizer.bpecsng}tt|}tj|D]#}dfdd|dD}|fdd | dDq|S)Nrc3|]}j|VqdSre)r)rrr+rr :sz)SimpleTokenizer.encode..r c3r8re)r)r bpe_tokenr+rrr9;s  r) rrlowerrfindallr#rencoderr7r )rRr bpe_tokensr0rr+rr=6s zSimpleTokenizer.encodecsDdfdd|D}tfdd|Djddddd }|S) Nrcg|]}j|qSr)r)rr0r+rrrArz*SimpleTokenizer.decode..cr?r)r)rcr+rrrCrr replace)errorsr r)r bytearrayrrA)rRrrrr+rr@szSimpleTokenizer.decodeNcs|sj}t|tr |g}jdjd}fdd|D}tjt||tjd}t|D]\}}|d|d|g}t |||dt|f<q0t|dkrW|dS|S)Nrrcsg|] }g|qSr)r=)rrrR sot_tokenrrrRsz,SimpleTokenizer.__call__..)dtyperr) r isinstancerrrrWr?long enumeratetensor)rRtextsr eot_token all_tokensresultr5rrrDr__call__Is    zSimpleTokenizer.__call__)rre) rfrgrhrrOr7r=rrOrrrrrs )  rcsleZdZ    ddedededed ed eed ed dffd d Z e ddZ ddZ ddZZS)IMUPreprocessorirNr~ kernel_sizeimu_stemr]rZrmrrrocst||_||_|du|_||_||_tt d|d||||_ |jdkr9tt d|j|j|_ ||dSrM)rNrOrTr]rrmrSr,rtrrr6rWrr)rRrSrTr]rZrmrrrTrrrO_s   zIMUPreprocessor.__init__cCsztjj|jdd|dkr)|jd}|jdkr'tj|j|j|9_dSdS|dkr6|jjddSt d|)Nrrqr~rrrr) r,rrr6r]rmrrrr@rrrrrzs  zIMUPreprocessor.init_parameterscCs||||}|jdksJ|jd|jksJ|jd}|jdkr4|j|dd}tj ||fdd}|j r<||j }|Sr) rLrQrr)r]rmrrrrArr6)rRrrrrrrrrrs   z*IMUPreprocessor.tokenize_input_and_cls_poscCsX|d|j|jdddd}||d|dd}|||j}d|iid}|S)Nr$rr rr%rr)unfoldrSr3r/rvrrT)rRimu imu_tokensrrrrrds" zIMUPreprocessor.forward)rQrNr~)rfrgrhr0rKrrrrrOrrVrrrdrkrrrTrrP^s4  rP)Nr)r)/rrrr1 functoolsrtypingrrrrrnumpyr regexrrtorch.nnr,iopath.common.file_ior timm.layersrimagebind.models.helpersr r r#r;rIrJrjrKrlr{rrrrrrrrrrobjectrrPrrrrsJ     , '#j P  m