čpe00dZddlZddlmZddlmZddlmZmZm Z m Z ddl m Z gdZ Gdd ejZGd d ejZGd d ejZGddejZGddeZGddejZdS)zZ EfficientNet, MobileNetV3, etc Blocks Hacked together by / Copyright 2020 Ross Wightman N) functional) create_conv2d drop_pathmake_divisiblecreate_act_layer)sigmoid) SqueezeExcite ConvBnActDepthwiseSeparableConvInvertedResidualCondConvResidual EdgeResidualcJeZdZdZddejejddffd ZdZxZ S)r a Squeeze-and-Excitation w/ specific features for EfficientNet/MobileNet family Args: in_chs (int): input channels to layer rd_ratio (float): ratio of squeeze reduction act_layer (nn.Module): activation layer of containing block gate_layer (Callable): attention gate function force_act_layer (nn.Module): override block's activation fn if this is set/bound rd_round_fn (Callable): specify a fn to calculate rounding of reduced chs g?NcXtt|||pt}|||z}|p|}t j||dd|_t|d|_t j||dd|_ t||_ dS)NrT)biasinplace) superr __init__roundnnConv2d conv_reduceract1 conv_expandgate) selfin_chsrd_ratio rd_channels act_layer gate_layerforce_act_layer rd_round_fn __class__s R/home/terry/CeaseFire/code_minimal_pytorch_ITTI/timm/models/efficientnet_blocks.pyrzSqueezeExcite.__init__s mT""++---  %.K%+fx&788K#0y 9V[!$GGG$Y=== 9[&!$GGG$Z00 c|dd}||}||}||}|||zS)N)T)keepdim)meanrrrr)rxx_ses r'forwardzSqueezeExcite.forward*savvfdv++%%yy%%499T??""r() __name__ __module__ __qualname____doc__rReLUSigmoidrr0 __classcell__r&s@r'r r sj  $(TRWz4T 1 1 1 1 1 1#######r(r cReZdZdZddddejejdffd ZdZdZ xZ S) r z@ Conv + Norm Layer + Activation w/ optional skip connection rFc tt||o |dko||k|_| |_t |||||||_| ||_|d|_dS)NrstridedilationpaddingTr) rr r has_residualdrop_path_raterconvbn1r) rrout_chs kernel_sizer>r?pad_typeskipr" norm_layerrBr&s r'rzConvBnAct.__init__5s i''))) FVq[FVw5F,!&';vX`jrsss :g&&Id+++ r(c|dkrtdd|jj}ntdd|jj}|S)N expansionrr0module hook_typenum_chsr:)dictrC out_channelsrlocationinfos r' feature_infozConvBnAct.feature_info?sF { " "vDIDZ[[[DDrR9OPPPD r(c|}||}||}||}|jr+|jdkrt ||j|j}||z }|SNr;)rCrDrrArBrtrainingrr.shortcuts r'r0zConvBnAct.forwardFsq IIaLL HHQKK IIaLL   "R''a!4dmDD MAr( r1r2r3r4rr5 BatchNorm2drrUr0r7r8s@r'r r 2s|891r"'bnUW,,,,,,       r(r c ZeZdZdZdddddddejejddf fd Zd Zd Z xZ S) r z DepthwiseSeparable block Used for DS convs in MobileNet-V1 and in the place of IR blocks that have no expansion (factor of 1.0). This is an alternative to having a IR with an optional first pw conv. r+rr:FNr;c tt||dko||ko| |_| |_| |_t ||||||d|_| ||_| d|_ | r | || ntj |_ t |||||_ | ||_|jr | dntj |_dS)NrTr>r?r@ depthwiserr"r@)rr rrA has_pw_actrBrconv_dwrDrrIdentityseconv_pwbn2act2)rrrEdw_kernel_sizer>r?rGnoskippw_kernel_sizepw_actr"rIse_layerrBr&s r'rzDepthwiseSeparableConv.__init__Ws $d++44666#q[>Vw->NJ ,$ FN6HV^jnppp :f%%Id+++ r?rGrk exp_ratioexp_kernel_sizerlr"rIrn conv_kwargsrBmid_chsr&s r'rzInvertedResidual.__init__sj %%..000!'R  )!344#w.>6Q;NJ,%VWoggxg[fgg :g&&Id+++ % Wn=5;h==0;== :g&&Id+++ =EW((7i8888"+--&gwhhPXh\ghh :g&&r(c|dkrtdd|jj}ntdd|jj}|SNrKrvrprLr:rPrvrqrQrRs r'rUzInvertedResidual.feature_infoF { " "z]DMLefffDDrR9STTTD r(c|}||}||}||}||}||}||}||}||}||}|j r+|j dkrt||j |j }||z }|SrW) rgrDrrdrhrirfrvrwrArBrrXrYs r'r0zInvertedResidual.forwards LLOO HHQKK IIaLL LLOO HHQKK IIaLL GGAJJ MM!   HHQKK   "R''a!4dmDD MAr(r[r8s@r'r r s34ATVC1XZX_~$WY'''''':r(r c XeZdZdZddddddddejejddd f fd Zd ZxZ S) rz, Inverted residual block w/ CondConv routingr+rr:FrtNrr;c||_t|j}tt|||||||| ||| | | | ||t j||j|_dS)N) num_experts) rjr>r?rGr"rkrxryrlrnrIrzrB)rrPrrrrLinear routing_fn)rrrErjr>r?rGrkrxryrlr"rIrnrrBrzr&s r'rzCondConvResidual.__init__s 't'7888  %%.. GN6T\go)Ud)Hal) / + + + )FD,<==r(c|}tj|dd}tj||}|||}||}||}| ||}| |}| |}| |}| ||}||}|jr+|jdkrt#||j|j}||z }|S)Nrr;)Fadaptive_avg_pool2dflattentorchr rrgrDrrdrhrirfrvrwrArBrrX)rr.rZ pooled_inputsrouting_weightss r'r0zCondConvResidual.forwards-a33;;A>> - (F(FGG LLO , , HHQKK IIaLL LLO , , HHQKK IIaLL GGAJJ MM!_ - - HHQKK   "R''a!4dmDD MAr() r1r2r3r4rr5r\rr0r7r8s@r'rrsy6634ATVC1XZX_~!TV>>>>>> r(rc \eZdZdZddddddddejejdd f fd Zd Zd Z xZ S) ra( Residual block with expansion convolution followed by pointwise-linear w/ stride Originally introduced in `EfficientNet-EdgeTPU: Creating Accelerator-Optimized Neural Networks with AutoML` - https://ai.googleblog.com/2019/08/efficientnet-edgetpu-creating.html This layer is also called FusedMBConv in the MobileDet, EfficientNet-X, and EfficientNet-V2 papers * MobileDet - https://arxiv.org/abs/2004.14525 * EfficientNet-X - https://arxiv.org/abs/2102.05610 * EfficientNet-V2 - https://arxiv.org/abs/2104.00298 r+rr:rFrtNr;ctt||dkrt|| z}nt|| z}| duo tdk}||ko|dko| |_||_t|||||||_| ||_ | d|_ | r | || ntj |_ t||| ||_| ||_dS) Nrr;rr=Trrarb)rrrrse_ratiorArBrconv_exprDrrrerfrvrh)rrrEryr>r?rG force_in_chsrkrxrlr"rIrnrBr{has_ser&s r'rzEdgeResidual.__init__s1 lD!!**,,, !  $\I%=>>GG$Vi%788G%7(R-#w.>6Q;NJ,& G_VhX`bbb :g&&Id+++ =EW((7i8888"+--&gwPXYYY :g&&r(c|dkrtdd|jj}ntdd|jj}|Sr}r~rRs r'rUzEdgeResidual.feature_info)rr(cj|}||}||}||}||}||}||}|jr+|jdkrt||j|j }||z }|SrW) rrDrrfrvrhrArBrrXrYs r'r0zEdgeResidual.forward0s MM!   HHQKK IIaLL GGAJJ MM!   HHQKK   "R''a!4dmDD MAr(r[r8s@r'rrs  45QUW5CUWU\~R''''''4r(r)r4rtorch.nnrrrlayersrrrrlayers.activationsr __all__Moduler r r r rrr(r'rs  $$$$$$NNNNNNNNNNNN'''''' t t t#####BI###B @22222RY222jGGGGGryGGGT/////'///dAAAAA29AAAAAr(