o yf!@sddlZddlZddlmZddlmZddlZddlZddlZddl m Z ddl m Z ddl mZmZmZmZmZmZmZdZdZed gd Zed Zd d ZddZGdddZdS)N) namedtuple)Path) transforms)InterpolationMode)devicespathssharedlowvram modelloadererrors torch_utilsizViT-L/14Categorynametopnitemsz \.top(\d+)$cCsddttjjdDS)NcSsg|]}|jqS)stem).0frrdC:\Users\Luke\Documents\Stable diffusion\Automatic1111\stable-diffusion-webui\modules\interrogate.py sz"category_types..*.txt)rr interrogator content_dirglobrrrrcategory_typessrc Cstd|d}gd}zPz'tj|dd|D]}tjd|dtj||dqt||Wnt yM}z t |dWYd}~nd}~wwWtj |r\t |dSdStj |rjt |ww) NzDownloading CLIP categories..._tmp)artistsZflavorsZmediumsZ movementsT)exist_okz`https://raw.githubusercontent.com/pharmapsychotic/clip-interrogator/main/clip_interrogator/data/z.txtz/downloading default CLIP interrogate categories)printosmakedirstorchhubdownload_url_to_filepathjoinrename Exceptionr displayexists removedirs)rtmpdirr category_typeerrr,download_default_clip_interrogate_categoriess$ (  r0c@seZdZdZdZdZdZdZddZddZ ddZ dd Z d d Z d d Z ddZddZddZdddZddZddZdS)InterrogateModelsNcCs(d|_g|_||_tjtdk|_dS)Ncpu)loaded_categoriesskip_categoriesrrdevice_interrogater#devicerunning_on_cpu)selfrrrr__init__4szInterrogateModels.__init__c Cs tj|js t|j|jdur|jtjj kr|jSg|_tj|jrtjj |_g}t |j dD]N}| |j |j |jvrCq4t|j }|durOdnt|d}t|ddd}dd|D}Wdn1sqwY|j t|j ||dq4|jS) Nrrutf8)encodingcSsg|]}|qSr)strip)rxrrrrMsz0InterrogateModels.categories..r)r!r&r+rr0r3r4ropts interrogate_clip_skip_categoriesrrappendrre_topnsearchintgroupopen readlinesr )r8rfilenamemrfilelinesrrr categories:s&     zInterrogateModels.categoriescCsGddd}|tjd<dS)Nc@seZdZddZdS)z>InterrogateModels.create_fake_fairscale..FakeFairscalecSsdSNrr8rrrcheckpoint_wrapperUszQInterrogateModels.create_fake_fairscale..FakeFairscale.checkpoint_wrapperN)__name__ __module__ __qualname__rPrrrr FakeFairscaleTs rTz.fairscale.nn.checkpoint.checkpoint_activations)sysmodules)r8rTrrrcreate_fake_fairscaleSsz'InterrogateModels.create_fake_fairscalec Csf|ddl}tjtjtjdddgdd}|j j |dt dtjtjddd d }| |S) NrBLIPzlhttps://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pthz.pthz$model_base_caption_capfilt_large.pth) model_path model_urlZ ext_filterZ download_namebaseconfigszmed_config.json) pretrained image_sizevitZ med_config) rWZ models.blipr Z load_modelsr!r&r'r models_pathblipZ blip_decoderblip_image_eval_sizeeval)r8modelsfiles blip_modelrrrload_blip_modelZs*z!InterrogateModels.load_blip_modelcCsZddl}|jr|jtdtjjd\}}n |jttjjd\}}||t j }||fS)Nrr2)r6 download_root)rh) clipr7loadclip_model_namercmd_optsclip_models_pathrctorr5)r8rimodel preprocessrrrload_clip_modeljs z!InterrogateModels.load_clip_modelcCs|jdur||_tjjs|js|j|_|jtj |_|j dur9| \|_ |_ tjjs9|js9|j |_ |j tj |_ t |j j|_dSrN)rfrgrrlno_halfr7halfrnrr5 clip_modelrqclip_preprocessr get_paramdtyperOrrrrjws     zInterrogateModels.loadcC.tjjs|jdur|jtj|_dSdSdSrN)rr@!interrogate_keep_models_in_memoryrtrnrr2rOrrrsend_clip_to_ram  z"InterrogateModels.send_clip_to_ramcCrxrN)rr@ryrfrnrr2rOrrrsend_blip_to_ramr{z"InterrogateModels.send_blip_to_ramcCs||tdSrN)rzr|rtorch_gcrOrrrunloads zInterrogateModels.unloadr:c sddl}ttjjdkrdttjjt|t}|j t dd tj }|j ||j}||jddd}tdtf tj }t|jdD]}|d||d|jjdd7}qS||jd}|j|dd\fd d t|DS) NrT)truncatedimkeepdimr:gY@)rcs4g|]}d|d|dfqS)rd)numpy)ri text_arrayZ top_labelsZ top_probsrrrs4z*InterrogateModels.rank..)rirr}rr@interrogate_clip_dict_limitrEminlentokenizelistrnr5rt encode_texttyperwnormr#zerosrangeshape unsqueezeTsoftmaxr2topk) r8image_featuresr top_countriZ text_tokens text_features similarityrrrrranks &zInterrogateModels.rankcCsttjttftjdttddg|d |j  t j }t|jj|dtjjtjjtjjd}Wd|dS1sFwY|dS)N) interpolation)g3<4'?gwg M?gy{ ?)gB91?gwt.?g U?rF)sample num_beams min_length max_length)rComposeResizerbrBICUBICToTensor Normalizerrrwrnrr5r#no_gradrfgeneraterr@interrogate_clip_num_beamsinterrogate_clip_min_lengthinterrogate_clip_max_length)r8 pil_imageZ gpu_imagecaptionrrrgenerate_captions  $ z"InterrogateModels.generate_captionc Csd}tjjddztt|||}| t|}| | d |j tj}tatM|j| |j }||jddd}|D]-}|j||j|jd}|D]\}} tjjr{|d |d | d d d 7}qd|d|7}qdqVWdn1swYWdn1swYWntytjddd|d7}Ynw|tj |S)N interrogate)jobrrTr)rz, (:rz.3f)z, zError interrogating)exc_infoz)!rstatebeginr Zsend_everything_to_cpurr}rjrr|rurrrwrnr5r#rautocastrt encode_imagerrMrrrr@interrogate_return_ranksr)r reportr~end) r8rresrZ clip_imagercatmatchesmatchscorerrrrsB       zInterrogateModels.interrogate)r:)rQrRrSrfrtrurwr7r9rMrWrgrqrjrzr|r~rrrrrrrr1-s$   r1)r!rU collectionsrpathlibrrer#Z torch.hub torchvisionr!torchvision.transforms.functionalrrVrrrr r r r rbrkr compilerCrr0r1rrrrs"    $