o yfX9@sddlmZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl Z ddl Z ddlmZmZddlZddlZddlmZddlmZddlZddlmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'dd lm(Z(dd l)m*Z*m+Z+m,Z,dd l-m.Z.m/Z/m0Z0ddl-m1Z1ddl2m3Z3ddl4Zddl5m6Z6ddl7Zddl8m9Z9ddl:m;Z;dd lm?Z?ddl@mAZAmBZBddlCmDZDmEZEdZFdZGddZHddZIddZJddZKd@ddZLddZMe dd Gd!d"d"ZNGd#d$d$ZOdAd&d'ZPGd(d)d)eQZRdBd*d+ZSd,d-ZTd.d/ZUd0d1ZVdCd2d3ZWdDd6d7ZXdDd8d9ZYd:d;ZZe dd Gdd?d?eNZ\dS)E) annotationsN) dataclassfield)ImageImageOps)exposure)Any) devices prompt_parsermasking sd_samplerslowvraminfotext_utilsextra_networks sd_vae_approxscriptssd_samplers_commonsd_uneterrorsrng)slerp) model_hijack)images_tensor_to_samplesdecode_first_stageapproximation_indexes)optscmd_optsstate)AddMiDaS)LatentDepth2ImageDiffusion)repeat rearrange) blendLayers BlendTypecCs&tdtt|tj}|S)NzCalibrating color correction.)logginginfocv2cvtColornpasarraycopy COLOR_RGB2LAB)imageZcorrection_targetr/cC:\Users\Luke\Documents\Stable diffusion\Automatic1111\stable-diffusion-webui\modules\processing.pysetup_color_correction+s r1c CsXtdtttjtt |tj |ddtj  d}t ||tj}|dS)NzApplying color correction.)Z channel_axisuint8RGB)r&r'r fromarrayr(r)rZmatch_histogramsr*r+r-Z COLOR_LAB2RGBastyper"r#Z LUMINOSITYconvert) correctionZoriginal_imager.r/r/r0apply_color_correction1s   r9cCs@|\}}}}td|}td|||}||||f|}|S)NRGBA)rnewimages resize_imagepaste)r.Z dest_size paste_locxywhZ base_imager/r/r0uncropAs  rEcCs^|dur ||fS|durt||j|jf|}|}|d}|||d}||fS)Nr:r4)r,rEwidthheightr7alpha_composite)r.r@overlayoriginal_denoised_imager/r/r0 apply_overlayKs    rKTcCsb|jdkr*|ddkr*|r|dddd}|S|dd}|S|d}|S)Nr:)rMLcSs|dkrdSdS)NrMrr/)rAr/r/r0]sz$create_binary_mask..)mode getextremasplitr7point)r.roundr/r/r0create_binary_maskZs rVcCs6|jjdvr1tj|jdd|||jdd}t|tt j }tj j j |ddd}||j}|S|jjd krJ|j|jdd |jjj|j|jd S|j}|d d}|dur|jd dkrtj|jdd|||jdd}t|tt j }tj j j |ddd}||j}|S|j|jddd d |j|jd S)N>concathybridrdevice?)rrrrr;r?)value crossattn-admr2)dtyper['diffusion_model.input_blocks.0.0.weightr; )modelconditioning_keytorchonesshaper[rrgetrsd_vae_encode_methodnn functionalpadtor` new_zerosnoise_augmentor time_embeddim state_dict)sd_modelrArFrGimage_conditioningsddiffusion_model_inputr/r/r0txt2img_image_conditioningds(    &      rxF)reprc@seZdZUdZded<dZded<dZded<dZded<dZded <dZ ded <dZ d ed <d Z ded<d Z ded<dZ ded<d Zded<d Zded<dZded<dZded<dZded<dZded<dZded<dZded<d Zded!<d"Zded#<d"Zded$<dZded%<dZded&<d'Zded(<d'Zded)<dZd*ed+<dZd,ed-<dZded.<d'Z ded/<dZ!ded0<dZ"ded1<dZ#ded2<dZ$ded3<dZ%ded4<dZ&ded5<dZ'ded6<dZ(d*ed7<dZ)ded8<dZ*ded9<dZ+ded:<dZ,ded;<dZ-dZ.d'Z/ded<<dZ0d=ed><e1dd'd?Z2d@edA<e1dd'd?Z3d,edB<e1d'd'd?Z4dedC<ddgZ5ddgZ6dZ7dDedE<e1dd'd?Z8dFedG<e1d'd'd?Z9dedH<e1dd'd?Z:dIedJ<e1d'd'd?Z;dedK<e1dd'd?ZdOedP<e1dd'd?Z?dedQ<e1dd'd?Z@d,edR<e1dd'd?ZAd,edS<e1dd'd?ZBd,edT<e1dd'd?ZCd,edU<e1dd'd?ZDd,edV<e1dd'd?ZEdedW<e1dd'd?ZFdedX<e1dd'd?ZGdedY<e1dd'd?ZHd,edZ<e1dd'd?ZId,ed[<e1dd'd?ZJd,ed\<e1dd'd?ZKd,ed]<e1dd'd?ZLdDed^<e1dd'd?ZMded_<e1dd'd?ZNded`<e1dd'd?ZOdeda<e1dd'd?ZPdedb<e1dd'd?ZQdedc<e1d'd'd?ZRdedd<dedfZSeTdgdhZejUdidhZeTdjdkZVeVjUdldkZVeTdmdnZWeWjUdodnZWdpdqZXdrdsZYddtduZZdvdwZ[dxdyZ\dzd{Z]dd|d}Z^dd~dZ_ddZ`ddZaddZbdddZcddZddddZedddZfddZgddZhddZidddZjdS)StableDiffusionProcessingNobjectrtstroutpath_samples outpath_gridspromptprompt_for_displaynegative_promptz list[str]stylesrLintseedsubseedrfloatsubseed_strengthseed_resize_from_hseed_resize_from_wTboolseed_enable_extras sampler_name schedulerr; batch_sizen_iter2stepsg@ cfg_scaleirFrG restore_facestilingFdo_not_save_samplesdo_not_save_gridzdict[str, Any]extra_generation_paramslistoverlay_imagesetado_not_reload_embeddingsdenoising_strengthddim_discretize s_min_unconds_churns_tmaxs_tmins_noiseoverride_settings$override_settings_restore_afterwards sampler_indexrefiner_checkpointrefiner_switch_atdisable_extra_networksrfirstpass_imagedefaultinitzscripts.ScriptRunner scripts_valuescript_args_valuescripts_setup_completedictcommentsz!sd_samplers_common.Sampler | Nonesampler is_using_inpainting_conditioning tuple | Nonepaste_to is_hr_passtuplecuczrng.ImageRNG | Nonerstep_multipliercolor_corrections all_promptsall_negative_prompts all_seeds all_subseeds iteration main_promptmain_negative_promptpromptsnegative_promptsseedssubseedsextra_network_datauser sd_model_name sd_model_hash sd_vae_name sd_vae_hashis_apicCs|jdur tdtjdi|_|jdurg|_d|_|jdur"|jntj|_|j dur.|j ntj |_ |j dur:|j ntj |_ |j durF|j ntj pLt d|_ |j durV|j ntj |_ |jp^i|_|jpdi|_|jpji|_d|_|js~d|_d|_d|_d|_tj|_tj|_dS)Nz[sampler_index argument for StableDiffusionProcessing does not do anything; use sampler_name)fileinfrLr)rprintsysstderrrr sampler_noise_scheduler_overriderrrrrrrrr script_argsrefiner_checkpoint_inforrrrrrz cached_uccached_cselfr/r/r0 __post_init__s,       z'StableDiffusionProcessing.__post_init__cCstjSN)sharedrtrr/r/r0rtz"StableDiffusionProcessing.sd_modelcCdSrr/rr^r/r/r0rt scC|jSr)rrr/r/r0rrz!StableDiffusionProcessing.scriptscCs0||_|jr|jr|js|dSdSdSdSr)rrr setup_scriptsrr/r/r0r  cCrr)rrr/r/r0rrz%StableDiffusionProcessing.script_argscCs0||_|jr|jr|js|dSdSdSdSr)rrrrrr/r/r0rrcCsd|_|jj||j ddS)NT)is_ui)rr setup_scripsrrr/r/r0r$sz'StableDiffusionProcessing.setup_scriptscCsd|j|<dSNr;)r)rtextr/r/r0comment)sz!StableDiffusionProcessing.commentcCs,|jjjdv|_t|j||p|j|p|jS)N>rWrX)rtrdrerrxrFrG)rrArFrGr/r/r0rx,sz4StableDiffusionProcessing.txt2img_image_conditioningc Cstdd}|dt|ddi}t|ddjtjd}t|d |jd }t |d d t t j }tjjj|j||jd dd dd}t|\}}d||||d}|S)NZ dpt_hybrid) model_typejpgrzc h w -> h w cmidas_in)N.rZz1 ... -> n ...)nr\r2bicubicF)sizerQ align_corners@r])rr!rf from_numpyrnrr[r rrrrirrjrkrl interpolatert depth_modelrhaminmax) r source_image transformer transformedrconditioning_image conditioning depth_min depth_maxr/r/r0depth2img_image_conditioning1s   z6StableDiffusionProcessing.depth2img_image_conditioningcCstj|}|Sr)rrtencode_first_stagerQ)rrrr/r/r0edit_image_conditioningDsz1StableDiffusionProcessing.edit_image_conditioningcCsd|j|}|jjdur0d}|jj|tt|g|jd|jddd\}}t ||fd}|S)Nrz1 -> b)b) noise_levelr;) rtembedderrpr rftensorrnr[rhcat)rrc_admrnoise_level_embr/r/r0unclip_image_conditioningIs  4z3StableDiffusionProcessing.unclip_image_conditioningcCs d|_|dur.t|r|}n.t|d}|tjd}t|d}|r-t |}n|j ddg|j ddR}|j |j |jd}t||d|t|d tjj}|j|j|}tjjj||j ddd }||j d d d d }tj||gdd }| tj |jj}|S)NTrNo@NNr;)r[r`r]inpainting_mask_weight)rrrL)rr)rrf is_tensorr*arrayr7r6float32rrUnew_onesrhrnr[r`lerpgetattrrrr rtget_first_stage_encodingrrkrlrexpandrtype)rr latent_image image_maskround_image_maskZconditioning_maskrrur/r/r0inpainting_image_conditioningQs.   z7StableDiffusionProcessing.inpainting_image_conditioningcCst|}t|jtr||S|jjdkr||S|jj dvr*|j ||||dS|jj dkr5| |S|jj j j}|dd}|durV|jddkrV|j |||dS||jd d ddS) Nedit>rWrX)rrr_rar;rb)rrrc)r cond_cast_float isinstancertrrcond_stage_keyrrrerrZ model_wrap inner_modelrdrsrirhro)rrrrrrvrwr/r/r0img2img_image_conditioningxs         z4StableDiffusionProcessing.img2img_image_conditioningcCrrr/)rrrrr/r/r0rszStableDiffusionProcessing.initcCstr)NotImplementedError)rrunconditional_conditioningrrrrr/r/r0samplesz StableDiffusionProcessing.samplecCs4d|_d|_d|_tjsddgt_ddgt_dSdSr)rrrrpersistent_cond_cacherzrrrr/r/r0closes zStableDiffusionProcessing.closecCs(|r|jp tjp |jp tjS|jptjSr)token_merging_ratio_hrrtoken_merging_ratiorfor_hrr/r/r0get_token_merging_ratios z1StableDiffusionProcessing.get_token_merging_ratiocstjtr j_ntjtrjgtj_n jjjg_tjtr2j_n jgtj_tjtjkrWt dtjdtjdfddjD_fddjD_jd_ jd_ dS)Nz(Received a different number of prompts (z) and negative prompts ()cg|] }tj|jqSr/r prompt_stylesapply_styles_to_promptr.0rArr/r0 z;StableDiffusionProcessing.setup_prompts..cr)r/rr+apply_negative_styles_to_promptrr-rr/r0r/r0r) rrrrrlenrrr RuntimeErrorrrrr/rr0 setup_promptss     " z'StableDiffusionProcessing.setup_promptscCs4||||tjtjj|tjtj|j|jtj tj tj fS)z| | dkr>| dSq*|d} t|tj |||tjj| d<Wdn1s_wY| | d<| dS)a Returns the result of calling function(shared.sd_model, required_prompts, steps) using a cache to store the result if the same arguments have been used before. cache is an array containing two elements. The first element is a tuple representing the previously used arguments, or None if no arguments have been used before. The second element is where the previously computed result is stored. caches is a list with items described above. FTzOld prompt editing timelinesrNr;) rrr?r Z)get_learned_conditioning_prompt_schedulesrr@r autocastrt) rfunctionr=rcachesrr>Z old_schedulesZ new_schedulesr@cacher/r/r0get_conds_with_cachings     z0StableDiffusionProcessing.get_conds_with_cachingcCstj|j|j|jd}tj|j|j|jdd}t|j}|r%| |j n|j }||j |_ ||_ | tj|||jg|j|_| tj|||jg|j|_dSN)rFrGT)rFrGZis_negative_prompt)r SdConditioningrrFrGrr find_sampler_configr total_stepsrrfirstpass_stepsrEget_learned_conditioningrrr"get_multicond_learned_conditioningrr)rrrsampler_configrIr/r/r0 setup_condss   z%StableDiffusionProcessing.setup_condscCs |j|jfSr)rrrr/r/r0 get_condss z#StableDiffusionProcessing.get_condscCst|j\|_|_dSr)r parse_promptsrrrr/r/r0parse_extra_network_promptssz5StableDiffusionProcessing.parse_extra_network_promptsreturncCs$tjo|j otjptj otj S)z;Returns whether generated images need to be written to disk)r samples_saversave_incomplete_imagesr interruptedskippedrr/r/r0 save_sampless$z&StableDiffusionProcessing.save_samplesr)NTFNFr)rRr)k__name__ __module__ __qualname__rt__annotations__r}r~rrrrrrrrrrrrrrrrrFrGrrrrrrrrrrrrrrrrrrrrr$r#rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrpropertysetterrrrrrxrrrrrrr r"r'r5r@rErNrOrQrWr/r/r/r0rzs                                                     '   ! rzc@s4eZdZddddZd d Zdd d ZdddZdS) ProcessedrLrNrprzc Csr||_|j|_|j|_||_||_|j|_||_ddd|jD|_|j |_ |j |_ |j |_ |j |_ t |dd|_|j|_|j|_|j|_|jrMtjnd|_|j|_|j|_|j|_|j|_|j|_|j|_t |dd|_|j|_| |_|j|_tj|_tj |_!|j"|_"|j#|_#|j$|_$|j%|_%|j&|_&|j'|_'|j(|_(|j)|_)|j*|_*|j+|_+t,|jt-s|jn|jd|_t,|jt-s|jn|jd|_|jdurt.t,|jt-s|jn|jdnd|_|jdurt.t,|jt-s|jn|jdnd|_|j/|_/|p|j0p|jg|_0|p|j1p|jg|_1|p|j2p|jg|_2| p*|j3p*|jg|_3| p1|g|_4t5|_6dS)Nrcss|]}|dVqdS) Nr/)r.rr/r/r0 sz%Processed.__init__..image_cfg_scalerrrL)7r=rrrrrr'joinrrFrGrrrrdrrrrface_restoration_modelrrrrrrrrindex_of_first_imagerr job_timestampr6 clip_skipr$r#rrrrrrrrrrrrrrrr infotextsprogram_versionversion) rra images_listrr'rrrrrrgrjrr/r/r0__init__ sd00 zProcessed.__init__cCsid|jdd|jd|jdd|jd|jd|jd|jd |jd |jd |jd |jd |j d|j d|j d|j d|j d|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|jd}tj|dddS)NrrrrrrrrrrrFrGrrrrrrf)rrrrrrrrrgrjrrhrirrlcSrrr/)or/r/r0rPcszProcessed.js..)r) rrrrrrrrFrGrrrrrrfrrrrrrrrrgrjrrhrirrljsondumps)robjr/r/r0js?sh       #z Processed.jsc Cs(t||j|j|jg||j||jdS)N)rposition_in_batchr)create_infotextrrrr)rraindexr/r/r0infotextes(zProcessed.infotextFcCs|r|jS|jSr)r#r$r%r/r/r0r'hsz!Processed.get_token_merging_ratio) rLrNNNNNrNr)rarzrX)rZr[r\rnrsrwr'r/r/r/r0r`s  6 &r`cCstj||||||d}|S)Nrrrr)rImageRNGnext)rhrrrrrragr/r/r0create_random_tensorslsr}c@seZdZdZdS)DecodedSamplesTN)rZr[r\already_decodedr/r/r/r0r~qsr~c Cs0t}t|jdD]}t||||dd}|rzt|dWnctjy}zVtjj r:t j }d} d} d} ntjj rHt j }d} d} d } n|tj|krQ|td | d | d | |t_|jtj|tj}t||||dd}WYd}~nd}~ww|dur||}||q |S) Nrr;vaebfloat16z%Automatically convert VAE to bfloat16rz 32-bit floatz)Automatically revert VAE to 32-bit floatszE To always start with 32-bit VAE, use --no-half-vae commandline flag.zMA tensor with all NaNs was produced in VAE. Web UI will now convert VAE into z3 and retry. To disable this behavior, disable the 'z ' setting.)r~rangerhrr test_for_nans NansExceptionrrauto_vae_precision_bfloat16rfrauto_vae_precisionr  dtype_vaerprint_error_explanationfirst_stage_modelrnappend) rdbatch target_devicecheck_for_nanssamplesir eZ autofix_dtypeZautofix_dtype_textZautofix_dtype_settingZautofix_dtype_commentr/r/r0decode_latent_batchusN  &  rcCs^|dks|dur d}nt|tr"zt|}Wn ty!d}Ynw|dkr-ttdS|S)NrrLl)rr|r Exceptionrandom randrange)rr/r/r0get_fixed_seeds   rcCst|j|_t|j|_dSr)rrrrar/r/r0fix_seeds rcCs ddl}|}|dkrd}|S)Nrz)launchgit_tag)rresr/r/r0rks rkc  Csz|rd}n |dur|||j}| dur|j} t|dtj} t|dd} |} |jdd} |r3|jn||}|r<|jn| |}tjdk}|rLt |}id|j d |j d |j d |jd t|d dd|rl|jdn||d|jrwtjndd|jd|jdtjr|jnddtjr|jnddtjrtjnddtjrtjnddtjr|jnddtjr|jndd|j dkrdn |r|j!dn||d|j dkrdn|j d|j"dks|j#dkrdn|j"d|j#|j$%d|j&rt|dt'jj(nd| dkrdn| |r tjnd| dkrdn| | r| dkrdn| t|ddtj)dkr.tj)nd|j*dkr7dn|j*|j+r?d ndd! |j$tj,rMt-ndtj.rU|j/ndd"}|0D]=\}}zt1|t2rp||||<nt3|r|d,it4||<Wq^t5yt6j7d#|d$dd%d||<Yq^wd&8d'd(|0D}|rd)|nd*}||d+|9S)-a this function is used to generate the infotext that is stored in the generated images, it's contains the parameters that are required to generate the imagee Args: p: StableDiffusionProcessing all_prompts: list[str] all_seeds: list[int] all_subseeds: list[int] comments: list[str] iteration: int position_in_batch: int use_main_prompt: bool index: int all_negative_prompts: list[str] Returns: str Extra generation params p.extra_generation_params dictionary allows for additional parameters to be added to the infotext this can be use by the base webui or extensions. To add a new entry, add a new key value pair, the dictionary key will be used as the key of the parameter in the infotext the value generation_params can be defined as: - str | None - List[str|None] - callable func(**kwargs) -> str | None When defined as a string, it will be used as without extra processing; this is this most common use case. Defining as a list allows for parameter that changes across images in the job, for example, the 'Seed' parameter. The list should have the same length as the total number of images in the entire job. Defining as a callable function allows parameter cannot be generated earlier or when extra logic is required. For example 'Hires prompt', due to reasons the hr_prompt might be changed by process in the pipeline or extensions and may vary across different images, defining as a static string or list would not work. The function takes locals() as **kwargs, as such will have access to variables like 'p' and 'index'. the base signature of the function should be: func(**kwargs) -> str | None optionally it can have additional arguments that will be used in the function: func(p, index, **kwargs) -> str | None note: for better future compatibility even though this function will have access to all variables in the locals(), it is recommended to only use the arguments present in the function signature of create_infotext. For actual implementation examples, see StableDiffusionProcessingTxt2Img.init > get_hr_prompt. rNri enable_hrFTr&StepsSamplerz Schedule typez CFG scalezImage CFG scalerdZSeedzFace restorationSizerAz Model hashModelz FP8 weightzCache FP16 weight for LoRAzVAE hashVAEzVariation seedzVariation seed strengthzSeed resize fromDenoising strengthr r; init_img_hashGPUTrue) rzConditional mask weightz Clip skipENSDzToken merging ratiozToken merging ratio hrzInit image hashRNGZNGMSTiling)VersionUserz!Error creating infotext for key "")exc_infoz, cSs8g|]\}}|dur||kr|n |dt|qS)Nz: )rquote)r.kvr/r/r0r/.s8z#create_infotext..z Negative prompt: rrbr/):rrrrr6r'rreta_noise_seed_deltarZ%is_sampler_using_eta_noise_seed_deltarrrrrrrfrFrGadd_model_hash_to_inforadd_model_name_to_inforr fp8r:r;add_vae_hash_to_inforadd_vae_name_to_inforrrrrrrirrr  randn_sourcerradd_version_to_infotextrkadd_user_name_to_inforitemsrrcallablelocalsrrreportrestrip)rarrrrrrtuse_main_promptrvrrirr$r# prompt_textrZ uses_ensdZgeneration_paramskeyr^Zgeneration_params_textZnegative_prompt_textr/r/r0rus-          $*  !  rurarRc CsH|jdur |j|dd|jD}zktj|jddur-|jddt|j D]\}}t j ||ddd|dkrGt|dkrOt q2t|j|t|}Wt|jd|jr| D]\}}tt |||dkr~t ql|St|jd|jr| D]\}}tt |||dkrt qww) NcSs6i|]}|tjvr||tjvrtj|nt|qSr/)rdata get_default)r.rr/r/r0 9s6z"process_images..sd_model_checkpointTF)r run_callbackssd_vaer)rbefore_processrkeys sd_modelsZcheckpoint_aliasesripopreload_model_weightsrrsetrreload_vae_weightsapply_token_mergingrtr'process_images_innerrsetattr)raZ stored_optsrrrr/r/r0process_images5sB     rcsf tjtrtjdksJnjdusJttjtjj dur.t j _ j dur7t j _ j dvrPtj _jdurPtdj tjjj_tjj_t_t_tjj j tjj!"ttr_#nfddt$tj%D_#ttr_&nfddt$tj%D_&t'j()t*j+rj,stj-.j/durˆj/0g}g}t12j3t4#5j%j#j&tj j6rt j7dkrt89t:;Wdn1swYtt<_=t$j>D]}|_?td kr=d|d dj>tj<_gthjtjirLtjnt4jkjljmjHjIjPjEd}Wdn 1smwYj/durt/n|}j/o||jp}tq|dd r|} nt jrdkrt jrj[d<tsj|tjtdd} t1u| v} t1jw| ddddd} ~txytjrtxztt<{j/durj/j|| |dj%|jD|d jD_EjF|jD|d jD_Gt/}t| } j/j~| |d| j} d5fdd } } t| D]\} }| _d!t|tdd"}|tj}j rn| r`t jr`tjt|jd#jH| jE| t j| | d$d% ttj |}tt|}j/durt/|}j/||j}tqd&d}tj jsd}ntqd'ddur| tjkrj| }nd}j/durt/| ||}j/||j|j}}jdur| tjkr| rt jrt|j|\}}tj|jd#jH| jE| t j| | d(d% tj| |}t|j|\}}j/dur&t/|}j/||j}| r@tj|jd#jH| jE| t j| | d)| | }||t jrR||jd*<|||durt jsdt jr|d+}| rt jrtj|jd#jH| jE| t j| | d,d% t jr||t jst jrt|d-d.td.|jtd"||jO|jMd/d-}| rt jrtj|jd#jH| jE| t j| | d0d% t jr||q~ tq|s|tbgddd_d}t|d"kot j}t jst jr^js^|s^t|jD}t jr>| dd1}|d|t jr6||jd*<|d|d }t jr^tj|jd2j#dj%dt j| dd1t j dd3 Wdn 1siwYWdn 1sywYjUsjXrtVjXttb|j#d|dj&d||d4}j/durj/||S)6zthis is the main loop that both txt2img and img2img use; it calls func_init once inside all the scopes and func_sample once per batchrN)NrNonenone$Could not find checkpoint with name cs&g|]}tjdkr|ndqS)r)rrr-)rarr/r0r/s&z(process_images_inner..csg|]}t|qSr/)rr-)rr/r0r/sz Approx NNrLFr;ry) batch_numberrrrz params.txtrCutf8)encodingzBatch z out of )rrrrrrrFullz VAE DecoderTrrr]rrxminmax)rc stjjj||jdS)N)rrvr)rurrrr)rvrrr/r0rwsz&process_images_inner..infotextrr2rz-before-face-restorationr'rasuffixmask_for_overlayrz-before-color-correction)r'ra parametersr4z-maskr:RGBarNz-mask-composite)rgrid)r'short_filenamerar)rmrr'rrgrj)rF)rrrr3r torch_gcrrrrrface_restorationrrrget_closet_checkpoint_matchrrrrtr7Zname_for_extrarrrZget_loaded_vae_namerZget_loaded_vae_hashrmodules sd_hijackrZapply_circularZclear_commentsr5rrrrospathexistsrembeddings_dirrZ embedding_dbZ!load_textual_inversion_embeddingsrprocessrfno_grad ema_scoperArlive_previews_enableshow_progress_typerrdrZ apply_unetr job_countrrrVrUstopping_generationrrrrrrrrrzopt_CrGopt_frFrrrbefore_process_batchrQrractivater process_batchrNrupdateno_prompt_historyopenrepaths data_pathr`writerwrrjobZapply_alpha_schedule_overrideunet_needs_upcastwithout_autocastr rrPostSampleArgs post_samplerrsd_vae_decode_methodrcpustackrclampr is_enabledsend_everything_to_cpunextjobpostprocess_batchPostprocessBatchListArgspostprocess_batch_listr=rW enumerate batch_indexr*moveaxisnumpyr6r3#save_images_before_face_restoration save_imagerr5r}samples_formatPostprocessImageArgspostprocess_imager.overlay_inpaintrPostProcessMaskOverlayArgspostprocess_maskoverlayr overlay_imager#save_images_before_color_correctionrKrr9!postprocess_image_after_compositerenable_pnginfor' return_mask save_maskr7return_mask_compositesave_mask_composite compositer<rr>grid_only_if_multiple return_grid grid_saver image_gridinsertr~ grid_formatgrid_extended_filenameZ deactivate postprocess)rarjZ output_imagesrr processedrZ samples_ddimpsZx_samples_ddimZ batch_paramsrwrWrx_sampler.pprrppmoZimage_without_cc_rJrrZimage_mask_compositergZ"unwanted_grid_because_of_img_countrrr/)rarrr0r]s                         4      $        6      "  0  .     0 @0      8S rcCsNd}||}t||}t||dd}t||dd}||fS)z2old algorithm for auto-calculating first pass sizei@)mathsqrtceil)rFrGZdesired_pixel_countZactual_pixel_countscaler/r/r0#old_hires_fix_first_pass_dimensionsxs r6cs^eZdZUdZded<dZded<dZded <dZded <d Zded <d Z ded<dZ ded<dZ ded<dZ ded<d Z ded<d Zded<d Zded<dZded<dZded<d Zded<d d gZd d gZed ddZded<edddZded<edddZded<edddZded<edddZded <ed ddZd!ed"<ed ddZded#<ed ddZd$ed%<ed ddZd$ed&<ed ddZd'ed(<ed ddZ d'ed)<ed ddZ!d'ed*<ed ddZ"d'ed+<ed ddZ#d'ed,<fd-d.Z$d/d0Z%d1d2Z&d3d4Z'd5d6Z(fd7d8Z)fd9d:Z*d;d<Z+fd=d>Z,fd?d@Z-fdAdBZ.Z/S)C StableDiffusionProcessingTxt2ImgFrr?rrrrfirstphase_widthfirstphase_heightrhr_scaleNr| hr_upscalerhr_second_pass_steps hr_resize_x hr_resize_yhr_checkpoint_namehr_sampler_name hr_schedulerr hr_prompthr_negative_prompt force_task_idrrhr_checkpoint_infohr_upscale_to_xhr_upscale_to_y truncate_x truncate_yrapplied_old_hires_behavior_tolatent_scale_moderhr_chr_ucrall_hr_promptsall_hr_negative_prompts hr_promptshr_negative_promptshr_extra_network_datacsRt|jdks|jdkr|j|_|j|_|j|_|j|_tj |_ tj |_ dS)Nr) superrr9r:rFrGrGrHr7 cached_hr_uc cached_hr_cr __class__r/r0rs  z.StableDiffusionProcessingTxt2Img.__post_init__cCstjr.|j|j|jfkr.|j|_|j|_|j|_|j|_t |j|j\|_|_|j|jf|_|jdkrR|jdkrR|j |j d<t |j|j |_t |j|j |_dS|jd|j|j d<|jdkrs|j|_|j|j|j|_dS|jdkr|j|j|j|_|j|_dS|j}|j}|j|j}|j|j}||kr|j|_|j|j|j|_n|j|j|j|_|j|_|j|t |_|j|t |_dS)Nrz Hires upscalerAz Hires resize)ruse_old_hires_fix_width_heightrKrFrGr>r?rGrHr6r;rrrrIrJ)rtarget_wtarget_hZ src_ratioZ dst_ratior/r/r0calculate_target_resolutions:      z.get_hr_promptc[r]r)rP)rarvrr^rDr/r/r0get_hr_negative_promptr`zEStableDiffusionProcessingTxt2Img.init..get_hr_negative_promptz Hires promptzHires negative promptzHires schedule typenearestc3s|] }|jjkVqdSr)namer<r-rr/r0rcsz8StableDiffusionProcessingTxt2Img.init..zcould not find upscaler named rLZtxt2img_upscaleFr2Tz Hires stepszHires upscaler)rrrr@rrrFrZ short_titlerArrBrr<rlatent_upscale_modesrilatent_upscale_default_moderLany sd_upscalersr\r processing_has_refined_job_countrrrr=r total_tqdm updateTotal)rrrrr_rarIr/rr0rsH       ,      3z%StableDiffusionProcessingTxt2Img.initc Cst|j|j|_|jdur||jr||jdur9t |j tj ddd}t |dd}d}t t|d} nzt |j tj d}t |dd}t tj|dd}|jtjtjd}tjdkrjtj|jd <t|ttj|j}d} tn7|j} |jj|| ||| | d }~ |js|St|jdurt !t"|j|tj#d d jt j d} nd} t$%t$j&|j'd Wdn1swY|(|| ||||S)Nrrr]r2raxisr`r VAE EncoderruTr)r'))r create_samplerrrtrrrrLr*r r6r rrfr+ expand_dimsrrnrr[r rrrjrrrrirrr{r rxrrrrZSkipWritingToConfigrrFsample_hr_pass) rrrrrrrr.rdecoded_samplesrAr/r/r0r s:       & z'StableDiffusionProcessingTxt2Img.samplec sztjjr|Sd_j}j}fdd} jpj} t | j _ j durkt |jdD]} | || q1tjjj||t|tfj dj dd}tdtjjd kretj ||} n|} ntj|d d d d d } g}t| D]C\} }d t|dd}| tj!}t"#|}| || t$j%d|||j&d}t'| tj(d }t|dd}|)|q}t*t'|}|j+tj,t-j.d}tj/dkrtj/j0d<t1|t23tj/}||} tj4|ddddj5d|jdj5ddj6d|jdj6ddf}t7j8|jddj9j:j;j<j=d_7j7>}t-?j@sZt-AtBCjDWdn 1sUwYt-A EWdn 1snwYtFGj jHddjIdurjIJj jK||jLjMjNpjO| d}tFGj Hd_ t-?tPj |t-jdd}d_|S)NTc sxrtjs dSt|tjstj||dd}tjj j gj |d}t j |jd||tj|dd dS)zsaves image before applying hires fix, if enabled in options; takes as an argument either an image or batch with latent space imagesNr) approximation)rrtrz-before-highres-fixr)rWrsave_images_before_highres_fixrrr sample_to_imagerurrrrr=rr}r)r.rvr'rrrr/r0save_intermediateJs  ,zJStableDiffusionProcessingTxt2Img.sample_hr_pass..save_intermediaterrQ antialias)rrQryr r]rrxrrr2)Z upscaler_namermrrnr;rYryr)rrurF)QrrrUrrGrHrArr rprtrrLrrhrfrkrlrrrrr rrrxrrr*rrrr6r3rr5r=r>r<r r rrrnr[r rrjrrrrir rJrIrrzrrrrrr{rrrArrrScalculate_hr_condsrrr'r before_hrsample_img2imgrMrNr=rr)rrrsrrrr target_width target_heightrxZimg2img_sampler_namerruZlowres_samples batch_imagesr-r.noiser/rwr0rrBsl   .         T,      (z/StableDiffusionProcessingTxt2Img.sample_hr_passcs8td|_d|_tjsddgt_ddgt_dSdSr) rTr"rMrNrr!r7rUrVrrWr/r0r"s  z&StableDiffusionProcessingTxt2Img.closecstjs dSjdkrj_jdkrj_tjtr'j_ n j j jg_ tjtr=j_ n j j jg_ fddj D_ fddj D_ dS)Nrcr)r/r*r-rr/r0r/r0zBStableDiffusionProcessingTxt2Img.setup_prompts..cr)r/r1r-rr/r0r/r0) rTr5rrCrrDrrrrOrrrPrrWrr0r5s       z.StableDiffusionProcessingTxt2Img.setup_promptscCs|jdurdStj|j|j|jd}tj|j|j|jdd}t|j p%|j }|j p,|j }|r4| |n|}|tj||j|j|jg|j||_|tj||j|j|jg|j||_dSrF)rMr rGrQrGrHrRr rHrArr=rrIrErKrJrUrrSrNrLrVr)rrQrRrMrrIr/r/r0rzs  $(z3StableDiffusionProcessingTxt2Img.calculate_hr_condscs|jr d|_|dStd|_d|_|jrt|jdurvtj j r)|dSt tj rxtj jtkrztt||jWdn1sMwY|tt||jWddS1smwYdSdSdSdSdSr)rrMrzrTrNrNrrFrrhires_fix_use_firstpass_condsr rrtr7rZselect_checkpointr rArrrSrrrWr/r0rNs(    "z,StableDiffusionProcessingTxt2Img.setup_condscs|jr |j|jfStSr)rrMrNrTrOrrWr/r0rOs  z*StableDiffusionProcessingTxt2Img.get_condscspt}|jr6|j|j|j|jd|j|_|j|j|j|jd|j|_t |j\|_|_ |Sr) rTrQrrOrrrQrPrRrrPrS)rrrWr/r0rQs $$zr?r@rArBrCrDrErUrVrrFrGrHrIrJrKrLrMrNrOrPrQrRrSrr\rr rrr"r5rzrNrOrQ __classcell__r/r/rWr0r7sV                 '70 _   r7cseZdZUdZded<dZded<dZded <dZded <dZd ed <d Z ded<d Z ded<dZ ded<dZ ded<dZ ded<dZded<dZded<dZded<dZded<dZded<dZded<edddZd ed<edddZd ed!<edddZd ed"<edddZded#<edddZded$<edddZd ed%<fd&d'Zed(d)Z e jd*d)Z d+d,Zd-d.Zd1d/d0Z Z!S)2 StableDiffusionProcessingImg2ImgNr init_imagesrr resize_moder8rrrdrmaskr$ mask_blur_x mask_blur_y mask_blurTr mask_roundinpainting_fillinpaint_full_resinpaint_full_res_paddinginpainting_mask_invertinitial_noise_multiplierr latent_maskr|rEFrrz torch.Tensornmaskrurr init_latentcs:t|j|_d|_|jdurtj|_dS|j|_dSr)rTrrrrrrrWr/r0rs "z.StableDiffusionProcessingImg2Img.__post_init__cCs|j|jkr |jSdSr)rrrr/r/r0rs z*StableDiffusionProcessingImg2Img.mask_blurcCst|tr ||_||_dSdSr)rrrrrr/r/r0r#s  c Cs|j|jd<tjjdkr|jnd|_t|j|j|_ d}|j }|dur$t ||j d}|j r9t|}d|jd<|jdkr_t|}dtd|jd d }t||d f|j}t|}|jdkrt|}dtd|jd d }t|d |f|j}t|}|jdks|jdkr|j|jd <|jr||_|d }t||j}|rt ||j!|j"|j!|j"}|\} } } } |#|}t$%d||j!|j"}| | | | | | f|_&d |jd<|j|jd<n?d}d}d|_d|_d} t'j()| t*+| n't$%|j,||j!|j"}t|}t-|.tj/ddd.tj0}t||_g|_1|j2dur-|j2n|}t3j4o7|j5du}|r>g|_5g}|j6D]}t3j7rat89|:;|_d|jrrrrr&r'rclipr6r r3rrrimg2img_color_correctionrr save_init_imghashlibmd5tobytes hexdigestrroutdir_init_imagesflattenimg2img_background_colorrr<r?rrr1rr3rqr rr4rfrrnr[r rrjrrrirrrkrlrrresizerharoundtiler+rr`rrr}rru)rrrrZ crop_regionrZnp_mask kernel_sizerx1y1x2y2massagerZadd_color_correctionsimgsimgr.Z image_maskedrZ init_maskZlatmaskr/r/r0r)s                  "   (              ( $ " :   "z%StableDiffusionProcessingImg2Img.initc Cs|j}|jdkr|j|jd<||j9}|jj||j||||jd}|jdurP||j |j|j} |j durNt ||j |j|j| } |j || | j } | }~t|S)Nr]zNoise multiplierro)rr{rrrr|rrurrr MaskBlendArgs on_mask_blendblended_latentr r) rrrrrrrrArZblended_samplesmbar/r/r0r s      z'StableDiffusionProcessingImg2Img.samplecCs"|jpd|jvr tjptjptjS)Nr$)r$rrtoken_merging_ratio_img2imgr%r/r/r0r's"z8StableDiffusionProcessingImg2Img.get_token_merging_ratiorX)"rZr[r\rr]rrrdrrrrrrrrrrrrErrrrurrrrr^r_rr r'rr/r/rWr0rs@                   r)T)NrxrrNrY)NrrFNN)rarzrRr`)] __future__rrpr&r2rrr dataclassesrrrfrr*PILrrrr(Zskimagertypingrmodules.sd_hijackrr r r r r rrrrrrrrZ modules.rngrrZmodules.sd_samplers_commonrrrZmodules.sharedrrrrZ modules.pathsrZmodules.face_restorationZmodules.imagesr=Zmodules.stylesmodules.sd_modelsrmodules.sd_vaerZ ldm.data.utilrldm.models.diffusion.ddpmreinopsr r!Zblendmodes.blendr"r#rrr1r9rErKrVrxrzr`r}rr~rrrrkrurrr6r7rr/r/r/r0s~   <           '~ d / s ( x