8gmddlZddlZddlmZddlmcmZddlZddlm Z m Z m Z m Z ddl mZddlmZmZmZddlmZddlZddlmZddlmZmZedZdd lmZdd l m!Z!dd l"m#Z#dd Z$ddlZd Z%GddeZ&y)N)invgeotrfnormalize_pointcloudclosed_form_inverse)mast3r_head_factory)CameraPredictorCameraPredictor_cleanMlp) FeatureNet)AsymmetricCroCo3DStereo)transpose_to_landscapefreeze_all_paramsinf)get_patch_embed) checkpoint)matrix_to_quaternionc|r td|tj|d}|djj dd}d|vr |ddd z}n"|j d d j d d }d |vsJ|rtd|t |}|j |dd}|r t||j|S)Nz... loading model fromcpu) map_locationargsManyAR_PatchEmbedPatchEmbedDust3Rlandscape_onlyz, landscape_only=False) zlandscape_only=Truezlandscape_only=Falsezinstantiating : modelFstrict)printtorchloadrreplaceevalload_state_dictto) model_pathdeviceverboseckptrnetss +/nas3/zsz/FLARE_huggingface/mast3r/model.py load_modelr.s & 3 ::ju 5D <   % %&9;M NDt#CRy44||C$,,-BDZ[ !T )) )  '( t*C DM%8A a 66&>cT|d|jdzz|jdzS)N) unsqueeze)xshiftscales r-modulater6%s) EOOA&& '%//!*< <#c}|_ tj|j|_!t1j2t5tE|jDcgc]Y}t1jFt1jHdt1j6|j:d|j:zd[c}|_%|jJD]\} t0jLjO| djPdt0jLjO| djRd^tj|j|_*tj|jV|_,tj|j|_-tj|j"|_.tj|j&|_/tj|j*|_0t1j2t5dDcgc].}t1j6|j8|j:d0c}|_1t1j2t5dDcgc]!}tj|j>#c}|_2tj|j>|_3tid|j:|j: |_5tm|_7t1jpd d dd d |_9t1jpd d dd d |_:t5dD]p}t0jLjO|jb|jPdt0jLjO|j<|jPdrgd|_;ty|jzd|_>|jdk(rtj|j|_?t0jLjO|j~jPdtid|j:|j: |_@t0jLjO|jjjPdt0jLjO|jjjRdyycc}wcc}wcc}wcc}wcc}w)NF)bias)inplaceTrr)hidden_features out_features@r1`g)activate)B desc_mode two_confsdesc_conf_modewogswposesuper__init__copydeepcopydec_blocks_finedec_blocks_pointcam_cond_encodercam_cond_encoder_point decoder_embeddecoder_embed_pointdec_normdec_norm_pointpose_token_refpose_token_ref_pointpose_token_sourcepose_token_source_pointcam_cond_embedcam_cond_embed_pointcam_cond_embed_point_prenn ModuleListrangeLinear enc_embed_dim dec_embed_dim inject_stage3enc_normenc_inject_stage3cam_cond_encoder_finelen SequentialSiLUadaLN_modulationinit constant_weightr;decoder_embed_fine dec_cam_normdec_cam_norm_fine dec_norm_finepose_token_ref_finepose_token_source_finecam_cond_embed_fine inject_stage2enc_inject_stage2enc_norm_coarser embed_poser cnn_wobnConv2dcnn_proj cnn_fusionidx_hookr encode_featureencode_feature_landscapedecoder_embed_stage2decoder_embed_fxfycxcyfc2) selfrNrMrJrKrLkwargsi_block __class__s r-rPzAsymmetricMASt3R.__init__)s"",   "6"!% d.B.B C&*mmD4I4I&J##'==1C1C#D "mmDMM:$(MM$2E2E$F!'+}}T5K5K'L$$(MM$2E2E$F!(, d6I6I(J%]]sxyzs{+|noBIId6H6H$J\J\ch,i+|}"$--W\]^W_0`RSt}}1M0`"a&*]]43H3H%I" " T1123/502}} GGE " IId((!d.@.@*@t L0 /5!6** 1E GG  eBi.. 2 GG  eBinna 0 1#'--0B0B"C!%t/@/@!A!]]4==9#'==1D1D#E &*mmD4J4J&K##'==1D1D#E ]]sxyzs{+|noBIId6H6H$J\J\ch,i+|}"$--W\]^W_0`RSt}}1M0`"a $}}T]];a1C1CRVRdRde#   "b!Q2 ))D"aA6q @A GG  d003::B ? GG  d003::B ? @" (>t?R?R]a(b% 99 (, d6H6H(ID % GG  d77>> C*-aASASbfbtbt*uD ' GG  d99==DDb I GG  d99==BBB G I,}0`/5,}0`s03[+ &[0A[53[:4&[?c :t|}t||fi|SN)dictrOr%)rr*kwnew_ckptrs r-load_state_dict_stage1z'AsymmetricMASt3R.load_state_dict_stage1es!:w&x6266r/c t|}|jdk(rJ|jD]7\}}d|vs |j}|||d|jd||||<9t ||fi|S)Ndpt_gsz dpt.head.4r)r head_typeitems state_dictshaperOr%)rr*rrkeyvaluerrs r-r%z AsymmetricMASt3R.load_state_dictjs: >>X %"jjl 4 U3&!%!2J7##Aq!Q/$,,Q1a8_MdMdefhiklnoMpqqr/c tjj|r t|dSt t ||fi|S)Nr)r()ospathisfiler.rOr8from_pretrained)clspretrained_model_name_or_pathrrs r-rz AsymmetricMASt3R.from_pretraineds< 77>>7 8;EJ J)3?@]dacd dr/cg}|j||\}}|jJ|jD]}|j||||}|j |}|||fS)N) true_shape) patch_embed enc_pos_embed enc_blocksappendri)rimagerinterm_featuresr3posblks r- _encode_imagezAsymmetricMASt3R._encode_images!!%J!?3!!)))?? C  " "1 %As A  MM! #&&r/c|Dcgc]}|d }}|Dcgc]}|d }}tj|d}|j\}}}}}|j} |jdg|jdd}tj|d}|jdg|jdd}|j ||\} } } | j | } tt| D]@} | | j | | | <| | j||g| jdd| | <B|}|dz}|dz}||z}| j||z||jjfj | }| j||z|dftj}|j\}}||k\}|}| j ||z|d z|d zdjd d dd}|jj!||jd dd dj#dd||<|jj!||j%dd jd dd dj#dd||<|j&j)d||| j*||<|j&j)d||| j*||<|j-|}|j||g|jddj | }|j||g|jdd}|j||g|jddd z}| j||g| jdd} | j||g| jdd} |j||g|jdd}||||| | | fScc}wcc}w) Nimgrr1dimrrCr@)dtyperHrr:)r!stackrrviewrr&rdrlr new_zerospatch_embed_coarse2 embed_dimint64Trprojflattenswapaxespatch_embed_test_position_getterr(r|)rviewsrimgsshapesBrrrroutrrrrn_tokensx_coarse pos_coarseheightwidth is_landscape is_portrait fine_token out_coarse shapes_coarses r-_encode_symmetrizedz$AsymmetricMASt3R._encode_symmetrizeds(-.U ..167$|$77 T* JJ5!Q tyy-djjn-f1-R3&,,qr"23$($6$6tV$D!S/ffUms?+, VA!0!3!6!6u!=OA !;!3!;!;Au!UsyyQRQS}!UOA  V  b bq5==!E'8T5M5M5W5W!XY\\]bc]]AeGXq#9]M "  #m XXagq1ua!eR8@@Aq!L !%!9!9!>!>z,?W!X!`!`abdeghjk!l!t!tuvxy!z $ 8 8 = =j>U>^>^_ace>f g o opqstvwyz {!D!DEFHI!J#'#9#9#I#I!QPQSVS]S]#^ < "&"8"8"H"HAqRUR\R\"] ;''1%X%%aD1CDGGN 'Z''5H:3C3CAB3GH &q%C&,,qr2BCqH ckk!U3SYYqr]3ckk!U3SYYqr]35<6<<+;<j*fc3WWE/7s O O ct|j||||_t|j|d|||_t|j|d|||_t|j|d|z||_y)NrC) input_dimrH)rpatch_embed_clsrpatch_embed_coarserr)rimg_size patch_sizerfs r-_set_patch_embedz!AsymmetricMASt3R._set_patch_embedsy*4+?+?:Wde"1$2F2FRSUbn{"|#243G3GSTVco|#} !01E1ExQRU_Q_an!or/c |d|zdk(r |d|zdk(s Jd|d|||_||_||_||_|j||_t |||t ||_t |||t ||_t|j||_ t|j||_ t|jjjd|j|_t|jjjd|j|_t d ||t ||_t|j$||_y) Nrr1z img_size=z must be multiple of patch_size=)has_confrFrH)hood_idx trunk_depthrsh) output_moder depth_mode conf_moderLrbooldownstream_head1downstream_head2r head1head2r dpthooksr pose_headpose_head_stage2downstream_head4head4) rrrrrrrrrs r-set_downstream_headz$AsymmetricMASt3R.set_downstream_heads_{Z'1, 22 2! S%.XK/PZM"R S!&"$"    &"+D  3I{D[_`i[j k 3I{D[_`i[j k+D,A,AN[ +D,A,AN[ .8M8M8Q8Q8W8Wefmqmvmvw 5t?T?T?X?X?^?^lmtxt}t} ~ 3D+tVZ[dVe f+D,A,AN[ r/c  tj||fd}tj||fd} |g} |j|}|j\} } } }|j | d|}| j | d| jd} g}| j ||j |}|j |}tj||fd}| dz } |jj|jj| ddj | d||jj|jj| | zddj | | zd|}}|j}d}tt|j|j|j |j"D]\}\}}}}||j%dd\}}}t'|j)| d||ddddfj)| d|ddddfj)| d}t'|j)| | zd||ddddfj)| | zd|ddddfj)| | zd}t+||| }|j | | dzd|}|ddddfj | d|}|ddddfj)| | zd|}tj||j | d|fd} tj||j | | zd|fd}!tj| |!fd}"t+||"}"|"ddddf}#|"ddddf}"|#d| j | d|}$|#| dj | | zd|}%|j | d||"d| j | d|z}|j | | zd||"| dj | | zd|z}|j ||f|j | d|||$z}|j | | zd|||%z}||j,vr|j | d||j.||j0|||dzddddfj | ddz}|j | | zd||j.||j0|||dzddddfj)| | zddz}|dz }|j | dd|}|j | | d|}tj||fd}| j ||j | d|}| d=|j3| d| d<t5t7|j8|d|d<| t|fS)Nr1rrr:rrC)r!rrsrrrr}rwr&rrrx enumerateziprSrkryrochunkr6rrrrzr{rvtuplemapru)&rf1pos1f2pos2pose1pose2 low_tokenfr final_outputrrrr cam_tokens pose1_embed pose2_embed pose_embedr[r]rhook_idxrblk1cam_condryro shift_msa scale_msagate_msafeatf1_camf2_camf_camf_deltaf_delta1f_delta2s& r-_decoder_stage2z AsymmetricMASt3R._decoder_stage2sU IIr2h "iit a(s  # #A &5!Q FF1b! hhq"ciim, Aooe, ooe, YY [91=  ,0,D,D,G,G,Q,X,XYZ[\]^,_,d,defhjlm,nptqLqLqOqOPRPXPXqYq`q`abchahijklqmqrqrstuzsz|~@AqB)JSTWX\XlXlnroIoIKOKcKceiezezU{K| ! FAFh 35E-=j-I-O-OPQWY-O-Z *Iy(%n&<&x+HHBDMM!WWQA&)E););H)EFfdF\F\]eFfgpqruvqvgwxyz|{|z|x|g}hChCDEGIKOhPGQ*RRWWQuWb!,/Kt/A/A(/KLlDLbLbckLlmvwx{|w|m}~ABACCnDnLnLMNOTMTVXZ^n_M`0aaA Ar1%BE2q)B 2r(A&A    "q"a A; !@ O--l2.>? Rs4#9#9:b>JK 2S*---r/c H|dg} | j|dtjjj dtj 5|j |} |j |} dddtj  fd} |dj\}}}}||j|d}n&|j|d|j|z}|dz }|j}|jj|j|ddj|d||j j|j||zddj||zd|}}tj||fd}|ntjjj dtj 5|j#|}|j#| } | |z} | | z} ddd| j|} | j|} || z}|| j||zd|z}d}t%t'|j(|j*|j,|j.|j0D]]\}\}}}}}||dzj3||dzd|ddddfj|d|}||dzj3||dzd|ddddfj3||zd|}|||z}|||z}tj|j|dd||j||d|fd} t5||j3||dzzd|| j3||dzzd||j3||dzzdd|j3||dzzdd\}!}"|!j||dzd|j3|d|}!t5||!|j3|dd}!|!j||dzd|}!|!ddddfj|d|}#|!ddddfj3||zd|}$tj||#j|d|fd}%tj||$j||zd|fd}&tj|%|&fd}'t5||'}'|'ddddf}(|'ddddf}'|(d|j|d|})|(|dj||zd|}*|j|d||'d|j|d|z}|j||zd||'|dj||zd|z}|#j|d|||)z}#|$j||zd|||*z}$||j6vr|#j|d||j8||j:|||dzddddfj|ddz}#|$j||zd||j8||j:|||dzddddfj3||zddz}$|dz }|#j|dd|}#|$j||d|}$tj|#|$fd}| j||j|d|}`| d=|j=| d| d<| S#1swYxYw#1swY xYw)Nrr1FenabledrrrCr)rr!cudaampautocastfloat32r}rrrXrrr\r&rrr^rrrrTdec_blocks_point_crossrVr`rarrrrhrjrZ)+rfeat_refrrrrr feat_stage2 fxfycxcy1 fxfycxcy2rrrr rrrrrrr[r]rr!rr blk_crossr#r`raf1_pref2_pref_prer'rrrr(r)r*r+r,r-s+ r-_decoder_stage3z AsymmetricMASt3R._decoder_stage3s  } HQK( ZZ^^ $ $U $ G 1//%0K//%0K 1YY [91= !"++5!Q  ((!5A((!58Q8QR]8^^A ,0,E,E,H,H,O,V,VWXYZ[\,],b,bcdfhjk,lnroKoKoNoNOToUo\o\]^_d]defghoiononopqvovxz|}o~)iit a(  ((U]](K 6 77 B  77 B )I5 )I5  6 "nnU+ !nnU+ '+5- 0@0@5"a0PP]fgjkolAlACGC^C^`d`{`{}A}V}VX\XuXuhv^w ! YAYY*>@Xac]**1uQwQ?"1"EJJ1bRSTFac]**1uQwQ?!"EMMaPUgWY[\]F6~FFF67HIIFIIv{{1aQ7QrST9UVXYZE AIIaqk2q,I5==YZ\abc\cYdfhjkKlnqnynyz{~CDE~E{FHJLMoNPSP[P[\]_def_f\gikmnPopGD!99QaQ/772qADc4QA)>?D99QaQ/DadB*Bad##AeGR3BYY2q0ABAFFYY 12771U7B3JKQOFIIvv.2Ex/EAabDkG!BQB$KEr{''2q1Hqr{''%Q7H+00B:U2AY^^ArST=UUN 1 6 6qwA FqrXYZ_X_acefIg g B"&:8&DDB5"a(+?+IIBDMM!WWQA&)E););H)EFfdF\F\]eFfgpqruvqvgwxyz|{|z|x|g}hChCDEGIKOhPGQ*RRWWQuWb!,/Kt/A/A(/KLlDLbLbckLlmvwx{|w|m}~ABACCnDnLnLMNOTMTVXZ^n_M`0aaA Ar1%BE2q)B 2r(A&A    "q"a AA !F O..|B/?@ R{ 1 1 6 6s#\  -\ \\!c :||fg}|j|}|j|}|j\}}}} |j|d| }|j|d| }|j|d|jd}|j|d|jd}g} |j||f|jj |j j|ddj|d| |jj |j j||zddj||zd| } } tt|j|j|j|jD]\} \}}}}t|g|dddd||\}}t|g|dddd||\}}t!j"| |j|d| fd}t!j"| |j||zd| fd}t!j"||fd}t||}|ddddf}|ddddf}|d|j|d| }||dj||zd| }| j|d| |d|j|d| z} | j||zd| ||dj||zd| z} | j| | f|j|d| ||z}|j||zd| ||z}|j|d| }|j|d| }|j||f|d=t%t'|j(| d| d<t| S)Nrr1r)rWrrrr[r&rrr]rr dec_blocks dec_blocks2rUr_rr!rrrrt)rrrrrrrrrrrr[r]rr"blk2r#r_rr(r)r*r+r,r-s r-_decoderzAsymmetricMASt3R._decoder_s*Rz    #    #5!Q WWQA  WWQA yyB 2/yyB 2/ RH%,0,?,?,B,B288,L,S,STUVWXY,Z,_,_`acegh,ikolBlBlElEFHFNFNlOlVlVWXY^W^_`ablclhlhijkpiprtvwlx)9B3tX\XhXhjnjjCGCVCVDW:X * 5A5dHntIl2&6tt&<IdIDIEBtIl2&6tt&<IdIDIEBYY2q0ABAFFYY 12771U7B3JKQOFIIvv.2Ex/EAabDkG!BQB$KEr{''2q1Hqr{''%Q7H+00B:U2AY^^ArST=UUN 1 6 6qwA FqrXYZ_X_acefIg g    ~/@A BB"^H%==B5"a(>(+CCBB"BB"B   R )' *, Os4#4#4jnEF 2Jr/c |ddj\}}}}t|}tjjj dtj 5|j||z\}} } } } } }ddd ddddfj|}| ddddfj|} ddddf}| ddddf}ddddf}|ddddf} ddddf}| ddddf} ddddf}| ddddf} ddddf}| ddddf}|j||||\}}|j|||d|\}}|||||||dd||f S#1swYxYw)NrrTr0r1interm_feature1interm_feature2r1r) rrlr!r2r3r4bfloat16rr&rDr)rview1view2r1r batch_sizerview_numrr'r shape_stage2r8 pos_stage2rfeat1feat2rrshape1shape2 shape1_stage2 shape2_stage2 feat1_stage2 feat2_stage2 pos1_stage2 pos2_stage2 pose_token1 pose_token2 pred_camerass r-forward_coarse_posez$AsymmetricMASt3R.forward_coarse_poses$Qx44 Aq!u: ZZ^^ $ $T $ H ~X\XpXpqvw|q|X} UFD#|[*o ~QU u%QU u%1bqb5z1ab5z2A212$QU+ $QU+ "1bqb5) "1ab5)  BQB'  AB' &*mmE4&M"k..[bmw{DI.J a[, \S`boquw{~IKVXgg g! ~ ~s E!!E*c|jr|j||||\}}}n|j||||\}}}|||fS)Nr0)rMforward_pointmap forward_gs)rrJrKr1rres1res2r\s r-forwardzAsymmetricMASt3R.forwardsV 99'+'<'This feature (novel view synthesis) has not been released yet.)NotImplementedError)rrJrKr1rs r-r`zAsymmetricMASt3R.forward_gss!"bccr/cVi}|jD]\}}dj|jdddzg|jdddz}|t|j j vslt d|d||||<|j||S)N.r _pose_headr1zLoading z from checkpointr)rjoinsplitrnamed_parameterskeysr r%)rr*rrrrnew_keys r-load_state_dict_poseheadz)AsymmetricMASt3R.load_state_dict_poseheads**, .JChh #q 1L @A399S>RSRTCUUVG$t4467<<>> )9:;$(I!  . ##HV#<}%|%j#|jd|%jd|%jd@c}%|$jdd}&|&j)d|&j+D]-}'|&|'j-d||d zfj |&|'</tj.j0j3dtj45|j7||"|#d tj4\}(}ddd|jdk(rv(ddj j j }|(ddj|dd j j j }n}}|jdd j!dd dz}||z }|})|}*tj|)|*fd}|dddd f}|ddd df} | (z}(|j9|!| | || |}+tj.j0j3dtj45|j'd|+D%cgc]=}%|%j jd|%jd|%jd?c}%|$jdd},ddd,j)d|,j+D]-}'|,|'j-d||d zfj |,|'</|&|,|(fScc}wcc}wcc}%w#1swYxYwcc}%w#1swYxYw)Nrrr0Frrr: quaternion_RrH camera_poser1T)rkeepdimrg:0yE>rCdescrF)rrlr]rNrdetachclonerr!rdoublerbmmrrnormmeanr&r._downstream_headpoprl unflattenr2r3r4r5r r?)-rrJrKr1rrLrrMrPrrQrpred_cameras_coarserRrS res1_stage1 res2_stage1rZr[rtransquaternion_R_predrref_camera_pose trajectory in_camera1gt_quaternion_Rgt_transsize trans_pred camera_embed camera_embed1 camera_embed2dec_finepose_token1_finepose_token2_finertokrarr\quaternion_R_noise trans_noisedec_fine_stage2rbs- r-r_z!AsymmetricMASt3R.forward_pointmapsO$Qx44 Aq!u:NRNfNfglns}DLQNfNR KtUD"5vv{T_alny|K :: '+C0668??AGGIEMM*b!4E 3B 7 G O OPZ\^`a b h h j q q s y y { #ii(Om)<(OQRSZZ\OEEM#RDD$7#RTUV]]_J,_=J:#4#4Z5E5Ea5H1#MzZJ 4Z2A2rr 5J K Q Q S [ [\fhjlm n q"1"ax(..088RKE/OHD1662t6LtST\ yy"3Z!@"E $QU+..u5 $QU+..u5 :>9M9MeUY[`bfhuxEGV:W66#%5 66*A.$$Qlt(ueh)>)>r399R=RUR[R[\^R_)`(uw|xExEFHJKxLM 99; QCS ++A HQJ/GHNNPDI Q ZZ^^ $ $U%-- $ H g"33JP`sCMQY^YfYf3gOL! g ::  $S)//188:@@BE ,R 0 @ H HUWYZ [ a a c j j l r r t  / ED1662t6LtST\ . yy"4k!BBG $QU+ $QU+ *\9 ..xt]Tacrs ZZ^^ $ $U%-- $ H V((n},~gjSYY[-@-@SYYr]TWT]T]^`Ta-b,~AFANANOQSTAUVD V 99; QCS ++A HQJ/GHNNPDI QT<''c)P#R")v  g g,- V Vs>1 W) W ,AW (W<W& AW! W&W!W&&W/)FTrxFN)rAir)NNNNT)__name__ __module__ __qualname__rPrr%r classmethodrrrrr r.r?rDr!rIr]rcr`rnr_ __classcell__)rs@r-r8r8(s:Hx7 7r.ee ' #XJp\(6.pAF% P9=ENNg,-1(045>>d=6:>(r/r8r)'r!rQtorch.nnrbtorch.nn.functional functionalrrdust3r.utils.geometryrrrrmast3r.catmlp_dpt_headrmast3r.vgg_pose_headrr r mast3r.shallow_cnnr mast3r.utils.path_to_dust3rmast3r dust3r.modelr dust3r.utils.miscr rrrdust3r.patch_embedrtorch.utils.checkpointr)pytorch3d.transforms.rotation_conversionsrr.r6r8rIr/r-rsb  XX6LL)"0G El.-J" =I(.I(r/