g5SrSSKrSSKJr SSKJr SSKJs Jr SSK J r J r SSK J r SSKJrJrJrJrJr SSKJr SS KJr SSKrSSKJr SS jr\"S S 9\"S S 9\"SS 9S.r"SS\R45r"SS\R45r"SS\R45r"SS\R45rSSjr\SSj5r \SSj5r!\SSj5r"g)aConViT Model @article{d2021convit, title={ConViT: Improving Vision Transformers with Soft Convolutional Inductive Biases}, author={d'Ascoli, St{'e}phane and Touvron, Hugo and Leavitt, Matthew and Morcos, Ari and Biroli, Giulio and Sagun, Levent}, journal={arXiv preprint arXiv:2103.10697}, year={2021} } Paper link: https://arxiv.org/abs/2103.10697 Original code: https://github.com/facebookresearch/convit, original copyright below N)partialIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)build_model_with_cfg)DropPath to_2tuple trunc_normal_ PatchEmbedMlp)register_model) HybridEmbedc .USSS[[SSSS. UE$)N)rTzpatch_embed.projhead) url num_classes input_size pool_sizemeanstdfixed_input_size first_conv classifierr)rkwargss 7C:\Users\Terry\Desktop\FGVC-HERBS\timm\models\convit.py_cfgr &s.=t%.BX\(     z5https://dl.fbaipublicfiles.com/convit/convit_tiny.pth)rz6https://dl.fbaipublicfiles.com/convit/convit_small.pthz5https://dl.fbaipublicfiles.com/convit/convit_base.pth) convit_tiny convit_small convit_basecr^\rSrSrS U4SjjrSrSrS SjrSrS\ S\ R4S jr S r U=r$) GPSA;cX>[TU]5 X lXlX-nUS-UlX`l[ R"XS-US9Ul[ R"XUS9Ul [ R"U5Ul [ R"X5Ul [ R"SU5Ul [ R"U5Ul[ R"[ R""UR55Ul[ R&"SSSS5Ulg)Nbiasrr)super__init__ num_headsdimscalelocality_strengthnnLinearqkvDropout attn_dropprojpos_proj proj_drop Parametertorchones gating_paramzeros rel_indices) selfr0r/qkv_biasr8r;r2head_dim __class__s rr. GPSA.__init__<s "#% !2))Cqx83(3I.IIc'  !Y/ I.LLDNN)CD).Q1a)@r!cURup#nURbURRSU:waURU5UlURU5nUR U5R X#UR X@R -5RSSSS5nXV-RSS5R X#U5nURU5nURU5nU$)Nrrr*r) shaperAget_rel_indices get_attentionr6reshaper/permute transposer9r;)rBxBNCattnr6s rforward GPSA.forwardOs''a    #t'7'7'='=a'@A'E#33A6D !!!$ FF1I  aDNNA4G H P PQRTUWXZ[ \ X A & . .qQ 7 IIaL NN1 r!cURup#nURU5RX#SURX@R-5R SSSSS5nUSUSpvUR R USSS5nURU5R SSSS5nXgRSS5-UR-n U RSS9n URSS9nURRSSSS5n S [R"U 5- U -[R"U 5U--n XRSS9R!S5-n UR#U 5n U $) Nr*rrrr0?)rHr5rKr/rLrAexpandr:rMr1softmaxr?viewr=sigmoidsum unsqueezer8) rBrNrOrPrQr5qk pos_score patch_scoregatingrRs rrJGPSA.get_attentionZsU''a WWQZ  ann9L M U UVWYZ\]_`bc d!ube1$$++Ar2r: MM),44Q1a@ ;;r2..$**< !))b)1 %%"%- ""''2q!4U]]6**k9EMM&h)rJrrAsqueezer=einsumsize)rBrN return_mapattn_map distancesdists rget_attention_mapGPSA.get_attention_mapjss%%a(--a0$$,,.q!Rx8B> ||K))>?)..QRBSS > !Kr!cURRRR[R "UR 55 Sn[URS-5nUS-S:XaUS- S- OUS-n[U5Hn[U5HnXBU--nSURRRUS4'SXC- -U-URRRUS4'SXS- -U-URRRUS4'M M URR=RUR-slg)Nrrhr*rrW) r6weightdatacopy_r=eyer0intr/ranger:r2)rBlocality_distance kernel_sizecenterh1h2positions r local_initGPSA.local_initss'    488!45$..B./ *5/Q*>+/Q&KSTDT $BK(b 009; $$))(A+69:bk9JM^9^ $$))(A+69:bk9JM^9^ $$))(A+6 )% !!T%;%;;!r! num_patchesreturncr[US-5n[R"SXS5n[R"U5R SS5[R"U5R SS5- nUR X"5nUR USS9R USS9nUS-US--nURS5USS2SS2SS2S4'URS5USS2SS2SS2S4'URS5USS2SS2SS2S4'URRRnURU5$)NrhrrrWrrYr*) rxr=r@aranger]repeatrepeat_interleaver`r5rtdeviceto) rBrimg_sizerAindindxindyinddrs rrIGPSA.get_rel_indicess{b()kk![qA ll8$))!R05<<3I3N3NrST3UUzz(-$$X1$5GGVWGXqy419$"&.."3 Aq!QJ"&.."3 Aq!QJ"&.."3 Aq!QJ&&~~f%%r!) r8r0r?r2r/r:r9r;r5rAr1r6)FrrZF)__name__ __module__ __qualname____firstlineno__r.rSrJrqrrxr=TensorrI__static_attributes__ __classcell__rEs@rr&r&;s?QS#%A&   < &3 &5<< & &r!r&c<^\rSrSrSU4SjjrSSjrSrSrU=r$)MHSAc&>[TU]5 X lX-nUS-Ul[R "XS-US9Ul[R"U5Ul[R "X5Ul [R"U5Ul g)Nr)rr+) r-r.r/r1r3r4qkvr7r8r9r;)rBr0r/rCr8r;rDrEs rr. MHSA.__init__sm "#% 99S'9I.IIc' I.r!cURup4nURU5RX4SURXPR-5R SSSSS5nUSUSUSpnXxR SS5-UR -n U RSS9RS5n [US -5n [R"U 5RSS5[R"U 5RSS5- n U RX5n U RU SS9RU SS9nU S-US--nUS -nURS 5n[R "S UU 45U- nU(aUU 4$U$) Nrr*rrrVrXrWrYrhcudari)rHrrKr/rLrMr1r\rrxr=rr]rrrrk)rBrNrmrOrPrQrrarbr6rnrrrrrrorps rrqMHSA.get_attention_mapsn''ahhqk!!!4>>1;NOWWXY[\^_abdefa&#a&#a&a B++tzz9###+003qBw<ll8$))!R05<<3I3N3NrST3UUzz(-$$X1$5GGVWGXqy419$BJ LL( ||K)X)>?!C > !Kr!cURup#nURU5RX#SURX@R-5R SSSSS5nUSUSUSpnXgR SS5-UR -n U RSS9n URU 5n X-R SS5RX#U5nURU5nURU5nU$) Nrr*rrrVrXrWrY) rHrrKr/rLrMr1r\r8r9r;) rBrNrOrPrQrrarbr6rRs rrS MHSA.forwards''ahhqk!!!4>>1;NOWWXY[\^_abdefa&#a&#a&aKKB''4::5|||#~~d# X A & . .qQ 7 IIaL NN1 r!)r8r/r9r;rr1)rFrrr) rrrrr.rqrSrrrs@rrrs /*  r!rcj^\rSrSrSSSSS\R \R S4U4SjjrSrSr U=r $) Block@FrTc  \>[T U]5 U "U5UlXlUR(a[ U4X$XeS.U D6UlO[ XXFUS9UlUS:a [U5O[R"5Ul U "U5Ul [X-5n [XXS9Ulg)N)r/rCr8r;r) in_featureshidden_features act_layerdrop)r-r.norm1use_gpsar&rRrr r3Identity drop_pathnorm2rxr mlp)rBr0r/ mlp_ratiorCrr8rr norm_layerrrmlp_hidden_dimrEs rr.Block.__init__s _  ==l(yldjlDISimnDI09B),BKKM_ S_-3R[gr!cXRURURU555-nXRURUR U555-nU$N)rrRrrrrBrNs rrS Block.forwardsK tyyA78 8 txx 1 67 7r!)rRrrrrr) rrrrr3GELU LayerNormr.rSrrrs@rrrs/13e"XZR\\TX hr!rc^\rSrSrSrSSSSSSSS S S S S S \R S SS S4U4SjjrSr\ RRS5r Sr SSjrSrSrSrU=r$)ConViTzHVision Transformer with support for patch or hybrid CNN input stage rrri rFrNrZTc>[TU]5 XW-nX@lUUlU=UlUlUUlUUlU b[XX5S9Ul O[XX5S9Ul URRnUUl [R"[R"SSU55Ul[R""U S9UlUR(aE[R"[R"SUU55Ul[)UR&SS9 [R*"SX5Vs/sHnUR-5PM nn[R."[1U5Vs/sH,nUU:a[3XWXXUUUSUS 9 O[3XWXXUUUS S 9 PM. sn5UlU"U5UlSS S 9/UlUS:a[R<"XT5O[R>"5Ul [)UR SS9 URCURD5 URG5H)unn[IUS5(dMURK5 M+ gs snfs snf)N)rin_chans embed_dim)r patch_sizerrr)p{Gz?rrT) r0r/rrCrr8rrrr2F) r0r/rrCrr8rrrr)num_chs reductionmoduler)&r-r.rlocal_up_to_layer num_featuresrr2 use_pos_embedr patch_embedr rr3r<r=r@ cls_tokenr7pos_drop pos_embedr linspaceitem ModuleListryrblocksnormdict feature_infor4rrapply _init_weights named_moduleshasattrr)rBrrrrrdepthr/rrC drop_rateattn_drop_ratedrop_path_ratehybrid_backboner global_poolrr2rrrNdprinmrEs rr.ConViT.__init__s-  &!2-66DN!2*  &*X \D  *!8 bD &&22 &ekk!Q &BC Y/   \\%++ai*PQDN $..c 2!&>!IJ!IAqvvx!IJmm5\ %#" $$ iCFWa"3  5 iCFWa " %# $ y) ")qPQ9DqBIIi5bkkm dnn#. 4%%&&&(DAqq,'' )+K %#s 5I..3I3c[U[R5(am[URSS9 [U[R5(a9UR b+[R RUR S5 ggg[U[R5(aU[R RUR S5 [R RURS5 gg)NrrrrZ) isinstancer3r4r rtr,init constant_r)rBrs rrConViT._init_weightss a # # !(( ,!RYY''AFF,>!!!&&!,-?' 2<< ( ( GG  affa ( GG  ahh ,)r!c SS1$)NrrrBs rno_weight_decayConViT.no_weight_decays [))r!cUR$r)rrs rget_classifierConViT.get_classifiers yyr!cXlUS:a'[R"URU5Ulg[R"5Ulg)Nr)rr3r4rrr)rBrrs rreset_classifierConViT.reset_classifiers4&>IAoBIIdnnk: SUS^S^S` r!cURSnURU5nURRUSS5nUR(aXR -nUR U5n[UR5H2upEX@R:Xa[R"X14SS9nU"U5nM4 URU5nUSS2S4$)NrrWrrY) rHrrr[rrr enumeraterrr=catr)rBrNrO cls_tokensublks rforward_featuresConViT.forward_features#s GGAJ   Q ^^**1b"5   NN"A MM!  ,FA***IIzo15AA- IIaLAwr!cJURU5nURU5nU$r)rrrs rrSConViT.forward5s$  ! !! $ IIaLr!)rrrrrrr2rrrrrrrr)rrrr__doc__r3rr.rr=jitignorerrrrrSrrrs@rrrs}!$QD\_gierZ\ "DR\\_c#$$3j- YY**a$r!rc zURSS5(a [S5e[[X4S[U0UD6$)N features_onlyzepsrr2rr/rr"rrrrrr3rrrr model_argsmodels rr"r"EsNKr $ ?KCIKJ  V=Z V: VE Lr!c p[S SSSS[[RSS9S.UD6n[ S SUS .UD6nU$) NrrZr rrrr#rrrrs rr#r#NsNKr $ ?KCIKJ  W>j WJ WE Lr!c p[S SSSS[[RSS9S.UD6n[ S SUS .UD6nU$) NrrZrrrrrr$rrrrs rr$r$WsNLr4!@LDJLJ  V=Z V: VE Lr!rr)#rr=torch.nnr3 functoolsrtorch.nn.functional functionalF timm.datarrhelpersrlayersr r r r r registryrvision_transformer_hybridrr rModuler&rrrrr"r#r$rr!rr-s $ A)GG$2  CE DF CE  Q&299Q&h-299-`BII.bRYYbJr!