Merge remote-tracking branch 'upstream/master' into ModelLoader

1 files changed, 62 insertions, 47 deletions

diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py
index cfc3ee40..5642b870 100644
--- a/modules/sd_samplers.py
+++ b/modules/sd_samplers.py
@@ -3,6 +3,7 @@ import numpy as np
 import torch

 import tqdm

 from PIL import Image

+import inspect

 

 import k_diffusion.sampling

 import ldm.models.diffusion.ddim

@@ -22,6 +23,8 @@ samplers_k_diffusion = [
     ('Heun', 'sample_heun', ['k_heun']),

     ('DPM2', 'sample_dpm_2', ['k_dpm_2']),

     ('DPM2 a', 'sample_dpm_2_ancestral', ['k_dpm_2_a']),

+    ('DPM fast', 'sample_dpm_fast', ['k_dpm_fast']),

+    ('DPM adaptive', 'sample_dpm_adaptive', ['k_dpm_ad']),

 ]

 

 samplers_data_k_diffusion = [

@@ -35,12 +38,12 @@ samplers = [
     SamplerData('DDIM', lambda model: VanillaStableDiffusionSampler(ldm.models.diffusion.ddim.DDIMSampler, model), []),

     SamplerData('PLMS', lambda model: VanillaStableDiffusionSampler(ldm.models.diffusion.plms.PLMSSampler, model), []),

 ]

-samplers_for_img2img = [x for x in samplers if x.name != 'PLMS']

+samplers_for_img2img = [x for x in samplers if x.name not in ['PLMS', 'DPM fast', 'DPM adaptive']]

 

 sampler_extra_params = {

-    'sample_euler':['s_churn','s_tmin','s_tmax','s_noise'],

-    'sample_heun' :['s_churn','s_tmin','s_tmax','s_noise'],

-    'sample_dpm_2':['s_churn','s_tmin','s_tmax','s_noise'],

+    'sample_euler': ['s_churn', 's_tmin', 's_tmax', 's_noise'],

+    'sample_heun': ['s_churn', 's_tmin', 's_tmax', 's_noise'],

+    'sample_dpm_2': ['s_churn', 's_tmin', 's_tmax', 's_noise'],

 }

 

 def setup_img2img_steps(p, steps=None):

@@ -98,6 +101,8 @@ class VanillaStableDiffusionSampler:
         self.init_latent = None

         self.sampler_noises = None

         self.step = 0

+        self.eta = None

+        self.default_eta = 0.0

 

     def number_of_needed_noises(self, p):

         return 0

@@ -120,20 +125,29 @@ class VanillaStableDiffusionSampler:
         self.step += 1

         return res

 

+    def initialize(self, p):

+        self.eta = p.eta or opts.eta_ddim

+

+        for fieldname in ['p_sample_ddim', 'p_sample_plms']:

+            if hasattr(self.sampler, fieldname):

+                setattr(self.sampler, fieldname, self.p_sample_ddim_hook)

+

+        self.mask = p.mask if hasattr(p, 'mask') else None

+        self.nmask = p.nmask if hasattr(p, 'nmask') else None

+

     def sample_img2img(self, p, x, noise, conditioning, unconditional_conditioning, steps=None):

         steps, t_enc = setup_img2img_steps(p, steps)

 

+        self.initialize(p)

+

         # existing code fails with cetain step counts, like 9

         try:

-            self.sampler.make_schedule(ddim_num_steps=steps,  ddim_eta=p.ddim_eta, ddim_discretize=p.ddim_discretize, verbose=False)

+            self.sampler.make_schedule(ddim_num_steps=steps,  ddim_eta=self.eta, ddim_discretize=p.ddim_discretize, verbose=False)

         except Exception:

-            self.sampler.make_schedule(ddim_num_steps=steps+1,ddim_eta=p.ddim_eta, ddim_discretize=p.ddim_discretize, verbose=False)

+            self.sampler.make_schedule(ddim_num_steps=steps+1, ddim_eta=self.eta, ddim_discretize=p.ddim_discretize, verbose=False)

 

         x1 = self.sampler.stochastic_encode(x, torch.tensor([t_enc] * int(x.shape[0])).to(shared.device), noise=noise)

 

-        self.sampler.p_sample_ddim = self.p_sample_ddim_hook

-        self.mask = p.mask if hasattr(p, 'mask') else None

-        self.nmask = p.nmask if hasattr(p, 'nmask') else None

         self.init_latent = x

         self.step = 0

 

@@ -142,11 +156,8 @@ class VanillaStableDiffusionSampler:
         return samples

 

     def sample(self, p, x, conditioning, unconditional_conditioning, steps=None):

-        for fieldname in ['p_sample_ddim', 'p_sample_plms']:

-            if hasattr(self.sampler, fieldname):

-                setattr(self.sampler, fieldname, self.p_sample_ddim_hook)

-        self.mask = None

-        self.nmask = None

+        self.initialize(p)

+

         self.init_latent = None

         self.step = 0

 

@@ -154,9 +165,9 @@ class VanillaStableDiffusionSampler:
 

         # existing code fails with cetin step counts, like 9

         try:

-            samples_ddim, _ = self.sampler.sample(S=steps, conditioning=conditioning, batch_size=int(x.shape[0]), shape=x[0].shape, verbose=False, unconditional_guidance_scale=p.cfg_scale, unconditional_conditioning=unconditional_conditioning, x_t=x, eta=p.ddim_eta)

+            samples_ddim, _ = self.sampler.sample(S=steps, conditioning=conditioning, batch_size=int(x.shape[0]), shape=x[0].shape, verbose=False, unconditional_guidance_scale=p.cfg_scale, unconditional_conditioning=unconditional_conditioning, x_T=x, eta=self.eta)

         except Exception:

-            samples_ddim, _ = self.sampler.sample(S=steps+1, conditioning=conditioning, batch_size=int(x.shape[0]), shape=x[0].shape, verbose=False, unconditional_guidance_scale=p.cfg_scale, unconditional_conditioning=unconditional_conditioning, x_t=x, eta=p.ddim_eta)

+            samples_ddim, _ = self.sampler.sample(S=steps+1, conditioning=conditioning, batch_size=int(x.shape[0]), shape=x[0].shape, verbose=False, unconditional_guidance_scale=p.cfg_scale, unconditional_conditioning=unconditional_conditioning, x_T=x, eta=self.eta)

 

         return samples_ddim

 

@@ -229,11 +240,13 @@ class KDiffusionSampler:
         self.model_wrap = k_diffusion.external.CompVisDenoiser(sd_model, quantize=shared.opts.enable_quantization)

         self.funcname = funcname

         self.func = getattr(k_diffusion.sampling, self.funcname)

-        self.extra_params = sampler_extra_params.get(funcname,[]) 

+        self.extra_params = sampler_extra_params.get(funcname, [])

         self.model_wrap_cfg = CFGDenoiser(self.model_wrap)

         self.sampler_noises = None

         self.sampler_noise_index = 0

         self.stop_at = None

+        self.eta = None

+        self.default_eta = 1.0

 

     def callback_state(self, d):

         store_latent(d["denoised"])

@@ -252,22 +265,12 @@ class KDiffusionSampler:
         self.sampler_noise_index += 1

         return res

 

-    def sample_img2img(self, p, x, noise, conditioning, unconditional_conditioning, steps=None):

-        steps, t_enc = setup_img2img_steps(p, steps)

-

-        sigmas = self.model_wrap.get_sigmas(steps)

-

-        noise = noise * sigmas[steps - t_enc - 1]

-

-        xi = x + noise

-

-        sigma_sched = sigmas[steps - t_enc - 1:]

-

+    def initialize(self, p):

         self.model_wrap_cfg.mask = p.mask if hasattr(p, 'mask') else None

         self.model_wrap_cfg.nmask = p.nmask if hasattr(p, 'nmask') else None

-        self.model_wrap_cfg.init_latent = x

         self.model_wrap.step = 0

         self.sampler_noise_index = 0

+        self.eta = p.eta or opts.eta_ancestral

 

         if hasattr(k_diffusion.sampling, 'trange'):

             k_diffusion.sampling.trange = lambda *args, **kwargs: extended_trange(self, *args, **kwargs)

@@ -276,33 +279,45 @@ class KDiffusionSampler:
             k_diffusion.sampling.torch = TorchHijack(self)

 

         extra_params_kwargs = {}

-        for val in self.extra_params:

-          if hasattr(p,val):

-            extra_params_kwargs[val] = getattr(p,val)

+        for param_name in self.extra_params:

+            if hasattr(p, param_name) and param_name in inspect.signature(self.func).parameters:

+                extra_params_kwargs[param_name] = getattr(p, param_name)

 

-        return self.func(self.model_wrap_cfg, xi, sigma_sched, extra_args={'cond': conditioning, 'uncond': unconditional_conditioning, 'cond_scale': p.cfg_scale}, disable=False, callback=self.callback_state, **extra_params_kwargs)

+        if 'eta' in inspect.signature(self.func).parameters:

+            extra_params_kwargs['eta'] = self.eta

 

-    def sample(self, p, x, conditioning, unconditional_conditioning, steps=None):

-        steps = steps or p.steps

+        return extra_params_kwargs

+

+    def sample_img2img(self, p, x, noise, conditioning, unconditional_conditioning, steps=None):

+        steps, t_enc = setup_img2img_steps(p, steps)

 

         sigmas = self.model_wrap.get_sigmas(steps)

-        x = x * sigmas[0]

 

-        self.model_wrap_cfg.step = 0

-        self.sampler_noise_index = 0

+        noise = noise * sigmas[steps - t_enc - 1]

+        xi = x + noise

 

-        if hasattr(k_diffusion.sampling, 'trange'):

-            k_diffusion.sampling.trange = lambda *args, **kwargs: extended_trange(self, *args, **kwargs)

+        extra_params_kwargs = self.initialize(p)

 

-        if self.sampler_noises is not None:

-            k_diffusion.sampling.torch = TorchHijack(self)

+        sigma_sched = sigmas[steps - t_enc - 1:]

 

-        extra_params_kwargs = {}

-        for val in self.extra_params:

-          if hasattr(p,val):

-            extra_params_kwargs[val] = getattr(p,val)

+        self.model_wrap_cfg.init_latent = x

 

-        samples = self.func(self.model_wrap_cfg, x, sigmas, extra_args={'cond': conditioning, 'uncond': unconditional_conditioning, 'cond_scale': p.cfg_scale}, disable=False, callback=self.callback_state, **extra_params_kwargs)

+        return self.func(self.model_wrap_cfg, xi, sigma_sched, extra_args={'cond': conditioning, 'uncond': unconditional_conditioning, 'cond_scale': p.cfg_scale}, disable=False, callback=self.callback_state, **extra_params_kwargs)

 

+    def sample(self, p, x, conditioning, unconditional_conditioning, steps=None):

+        steps = steps or p.steps

+

+        sigmas = self.model_wrap.get_sigmas(steps)

+        x = x * sigmas[0]

+

+        extra_params_kwargs = self.initialize(p)

+        if 'sigma_min' in inspect.signature(self.func).parameters:

+            extra_params_kwargs['sigma_min'] = self.model_wrap.sigmas[0].item()

+            extra_params_kwargs['sigma_max'] = self.model_wrap.sigmas[-1].item()

+            if 'n' in inspect.signature(self.func).parameters:

+                extra_params_kwargs['n'] = steps

+        else:

+            extra_params_kwargs['sigmas'] = sigmas

+        samples = self.func(self.model_wrap_cfg, x, extra_args={'cond': conditioning, 'uncond': unconditional_conditioning, 'cond_scale': p.cfg_scale}, disable=False, callback=self.callback_state, **extra_params_kwargs)

         return samples