o پiH @s~dZddlmZddlmZddlZddlmZddlmZm Z ddl m Z ddl m Z mZmZd d lmZd d lmZmZmZd gZGd d d e ZdpddZdqddZeideddddeddddeddddeddddd d!edd"d#d$d%edd&d#d$d'edd(d#d$d)edd*ddd#d+d,edd-dd.edd/ddd d0edd1dd2edd3dd4edd5ddd d6edd7dd8edd9ddd d:edd;ddedd?ddd edd@dd>eddAdd>eddBddd eddCdd>eddDddd eddEdd>dFZedrdGe fdHdIZedrdGe fdJdKZedrdGe fdLdMZedrdGe fdNdOZedrdGefdPdQZedrdGefdRdSZ edrdGefdTdUZ!edrdGefdVdWZ"edrdGe fdXdYZ#edrdGe fdZd[Z$edrdGe fd\d]Z%edrdGe fd^d_Z&edrdGe fd`daZ'edrdGe fdbdcZ(edrdGe fdddeZ)edrdGe fdfdgZ*ee+ds z)VisionTransformerDistilled.get_classifierNr$rcCsR||_|dkrt|j|nt|_|dkr"t|j|j|_dSt|_dS)Nr)r$rr%rr&r;r')r*r$rr/r/r0reset_classifierBs ,z+VisionTransformerDistilled.reset_classifierTcCs ||_dSr9)r()r*enabler/r/r0set_distilled_trainingGs z1VisionTransformerDistilled.set_distilled_trainingcCs|jr'|j\}}}}|jj}t|j||f||jrdn|jd}||d|}n|j}|jrN||}t j |j |jddd|j |jddd|fdd}n t j |j |jddd|j |jddd|fdd}||}||S)Nr)new_sizeold_sizerr )dim)dynamic_img_sizeshaper! grid_sizerr#no_embed_classrviewrcat cls_tokenexpandr pos_drop)r*xBHWCprev_grid_sizer#r/r/r0 _pos_embedKs: z%VisionTransformerDistilled._pos_embed pre_logitscCsp|dddf|dddf}}|r||dS||}||}|jr2|jr2tjs2||fS||dS)Nrr r)r;r'r(trainingrjit is_scripting)r*rMrTx_distr/r/r0 forward_headls"    z'VisionTransformerDistilled.forward_headrFr9)T)__name__ __module__ __qualname____doc__rr)rrVignorer7rModuler<intrstrr=r?rSboolTensorrY __classcell__r/r/r-r0rs   !FcKsF|dd}|r tnt}t|||fttddt|ddd|}|S)N out_indicesT)adapt_layer_scalegetter)rg feature_cls)pretrained_filter_fn feature_cfg)rrr r rr r5)variant pretrained distilledr,rg model_clsmodelr/r/r0 _create_deitzs    rsrc Ks |ddddddttddd |S) Ni)rhrtg?bicubicTzpatch_embed.projr;) urlr$ input_size pool_sizecrop_pct interpolationfixed_input_sizemeanr1 first_conv classifierr)rvr,r/r/r0_cfgsrzdeit_tiny_patch16_224.fb_in1kztimm/zFhttps://dl.fbaipublicfiles.com/deit/deit_tiny_patch16_224-a1311bcf.pth) hf_hub_idrvzdeit_small_patch16_224.fb_in1kzGhttps://dl.fbaipublicfiles.com/deit/deit_small_patch16_224-cd65a155.pthzdeit_base_patch16_224.fb_in1kzFhttps://dl.fbaipublicfiles.com/deit/deit_base_patch16_224-b5f2ef4d.pthzdeit_base_patch16_384.fb_in1kzFhttps://dl.fbaipublicfiles.com/deit/deit_base_patch16_384-8de9b5d1.pth)rhrg?)rrvrwryz'deit_tiny_distilled_patch16_224.fb_in1kzPhttps://dl.fbaipublicfiles.com/deit/deit_tiny_distilled_patch16_224-b40b3cf7.pthr:)rrvr~z(deit_small_distilled_patch16_224.fb_in1kzQhttps://dl.fbaipublicfiles.com/deit/deit_small_distilled_patch16_224-649709d9.pthz'deit_base_distilled_patch16_224.fb_in1kzPhttps://dl.fbaipublicfiles.com/deit/deit_base_distilled_patch16_224-df68dfff.pthz'deit_base_distilled_patch16_384.fb_in1kzPhttps://dl.fbaipublicfiles.com/deit/deit_base_distilled_patch16_384-d0272ac0.pth)rrvrwryr~zdeit3_small_patch16_224.fb_in1kz;https://dl.fbaipublicfiles.com/deit/deit_3_small_224_1k.pthzdeit3_small_patch16_384.fb_in1kz;https://dl.fbaipublicfiles.com/deit/deit_3_small_384_1k.pthz deit3_medium_patch16_224.fb_in1kzhttps://dl.fbaipublicfiles.com/deit/deit_3_huge_224_21k_v1.pth)(deit3_small_patch16_384.fb_in22k_ft_in1k)deit3_medium_patch16_224.fb_in22k_ft_in1k'deit3_base_patch16_224.fb_in22k_ft_in1k'deit3_base_patch16_384.fb_in22k_ft_in1k(deit3_large_patch16_224.fb_in22k_ft_in1k(deit3_large_patch16_384.fb_in22k_ft_in1k'deit3_huge_patch14_224.fb_in22k_ft_in1kr8cK2tddddd}td d|it|fi|}|S) z DeiT-tiny model @ 224x224 from paper (https://arxiv.org/abs/2012.12877). ImageNet-1k weights from https://github.com/facebookresearch/deit.  rh patch_sizerdepth num_headsdeit_tiny_patch16_224roN)rr5rsror, model_argsrrr/r/r0rrcKr) z DeiT-small model @ 224x224 from paper (https://arxiv.org/abs/2012.12877). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrrdeit_small_patch16_224roN)rrrr/r/r0rrrcK2tddddd}tdd|it|fi|}|S) z DeiT base model @ 224x224 from paper (https://arxiv.org/abs/2012.12877). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrdeit_base_patch16_224roN)rrrr/r/r0rrrcKr) z DeiT base model @ 384x384 from paper (https://arxiv.org/abs/2012.12877). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrrdeit_base_patch16_384roN)rrrr/r/r0rrrcK6tddddd}t d |ddt|fi|}|S) z DeiT-tiny distilled model @ 224x224 from paper (https://arxiv.org/abs/2012.12877). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrrhrdeit_tiny_distilled_patch16_224TrorpN)rrrr/r/r0rrcKr) z DeiT-small distilled model @ 224x224 from paper (https://arxiv.org/abs/2012.12877). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrrr deit_small_distilled_patch16_224TrN)rrrr/r/r0r'rrcK6tddddd}t d |ddt|fi|}|S) z DeiT-base distilled model @ 224x224 from paper (https://arxiv.org/abs/2012.12877). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrrdeit_base_distilled_patch16_224TrN)rrrr/r/r0r2rrcKr) z DeiT-base distilled model @ 384x384 from paper (https://arxiv.org/abs/2012.12877). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrrdeit_base_distilled_patch16_384TrN)rrrr/r/r0r=rrcK6tddddddd}td d |it|fi|}|S) z DeiT-3 small model @ 224x224 from paper (https://arxiv.org/abs/2204.07118). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrrTư>rrrrrG init_valuesdeit3_small_patch16_224roN)rrrr/r/r0rHrcKr) z DeiT-3 small model @ 384x384 from paper (https://arxiv.org/abs/2204.07118). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrrTrrdeit3_small_patch16_384roN)rrrr/r/r0rRrrcKr) z DeiT-3 medium model @ 224x224 (https://arxiv.org/abs/2012.12877). ImageNet-1k weights from https://github.com/facebookresearch/deit. rirTrrdeit3_medium_patch16_224roN)rrrr/r/r0r\rrcK6tddddddd}td d|it|fi|}|S) z DeiT-3 base model @ 224x224 from paper (https://arxiv.org/abs/2204.07118). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrTrrdeit3_base_patch16_224roN)rrrr/r/r0rfrrcKr) DeiT-3 base model @ 384x384 from paper (https://arxiv.org/abs/2204.07118). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrTrrdeit3_base_patch16_384roN)rrrr/r/r0rprrcK6tddddddd}td d|it|fi|}|S) z DeiT-3 large model @ 224x224 from paper (https://arxiv.org/abs/2204.07118). ImageNet-1k weights from https://github.com/facebookresearch/deit. rTrrdeit3_large_patch16_224roN)rrrr/r/r0rzrrcKr) z DeiT-3 large model @ 384x384 from paper (https://arxiv.org/abs/2204.07118). ImageNet-1k weights from https://github.com/facebookresearch/deit. rrrTrrdeit3_large_patch16_384roN)rrrr/r/r0rrrcKr) ri rTrrdeit3_huge_patch14_224roN)rrrr/r/r0rrrrrrrrrr) deit3_small_patch16_224_in21ft1k deit3_small_patch16_384_in21ft1k!deit3_medium_patch16_224_in21ft1kdeit3_base_patch16_224_in21ft1kdeit3_base_patch16_384_in21ft1k deit3_large_patch16_224_in21ft1k deit3_large_patch16_384_in21ft1kdeit3_huge_patch14_224_in21ft1k)FFrZr[),r_ functoolsrtypingrrr timm.datarr timm.layersrtimm.models.vision_transformerr r r _builderr _registryrrr__all__rrsr default_cfgsrrrrrrrrrrrrrrrrr\r/r/r/r0sx     ^   "%),/36:>B a