U Kf@sddlZddlZddlmZddlmmZddlZddlmZddl Z ddl m Z ddl Z ddl m Z ddlmZddlmZddlZddlZddlmZdd ZGd d d ZdS) N)tqdm) OrderedDict) defaultdict)DebertaV2ForMaskedLM)DebertaV2Tokenizer) VideoLoadercCs$dt|d|kdd}|S)z9 Computes a batch of padding masks given batched lengths r)torcharange unsqueeze transpose)lengths max_lengthmaskr/root/demo/Infer.pyget_masksrc@s,eZdZddZddZddZddZd S) InfercCstjddd}|d}d|_d|_||_tjd|dd |_tjd d d |_ t j|j r\|j nd |j |j|j|j|j|j|j|j|jd d |j|jd|_t}|dD]\}}|||dd<q|jj|d d|j|j|||_t|_ |!dS)Nzckpts/model.pthcpu) map_locationargszViT-L/14)devicerzckpts/deberta-v2-xlargeT)local_files_onlyF) features_dim max_feats freeze_lm freeze_mlmft_lnds_factor_attn ds_factor_ffdropoutn_ans freeze_lastZpretrained_model_name_or_pathradd_video_feat freeze_admodelzmodule.)strict)"r loadr# max_tokensrclip clip_modelrfrom_pretrained tokenizerrZ use_videorrrrrr r!r"r$r%r&r'ritemsreplaceload_state_dictevaltorr video_loader set_answer)selfrZpretrained_ckptrZnew_state_dictkvrrr__init__sF  zInfer.__init__cCs|j||j}|S)N)r-Z encode_imager4r)r7videofeatrrr_get_clip_featureEszInfer._get_clip_featurecCsxtj|jdddddddtjd}tj|jd ddddddtjd}t||g}|jj||jj|j j d dS) NYesFrTr)add_special_tokensr truncationpadding input_ids)dtypeNo)r$) r tensorr/longstackr'Zset_answer_embeddingsr4rrr$)r7Ztok_yesZtok_noZa2tokrrrr6Js:   zInfer.set_answerc Cs||\}}||d}t|d}tjtd|gdd}g}| }|ddkrlt |d}t t |D]} d|d || d |j jd } | } |j | d |jjd d dd} |j||j||j| d|j| d|jd} | d} d}| dd|| dd|f| d|j jk} || ddddfqxt|d} | jddkr| d}n | dj}||S)Nr )rrr)dim?z Question: z Is it 'z'? z . Subtitles: Tlongestpt)r?rrAr@Zreturn_tensorsrBattention_mask)r; video_maskrBrNlogits )r5r=r floatrr catones capitalizestripstrrangelenr/ mask_tokenrr+r'r4rsizeZ mask_token_idappendsoftmaxrGshaperoundrFsqueezemaxindices)r7text candidates video_pathr;Z video_lenrOZ logits_listquestionZaidpromptencodedoutputrPdelaypredsrrrgenerateesJ          zInfer.generateN)__name__ __module__ __qualname__r:r=r6rlrrrrrs&r)osr torch.nnnntorch.nn.functional functionalFmathrargparse collectionsrjsonrZmodel.deberta_moer transformersrr,ffmpegrrrrrrrs