Don't compute latent mask if were not using it. Also added support for fixed highres_fix generation.

1 files changed, 45 insertions, 27 deletions

diff --git a/modules/processing.py b/modules/processing.py
index a6c308f9..684e5833 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -541,12 +541,8 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
             self.truncate_y = int(self.firstphase_height - firstphase_height_truncated) // opt_f

 

 

-    def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength):

-        self.sampler = sd_samplers.create_sampler_with_index(sd_samplers.samplers, self.sampler_index, self.sd_model)

-

-        if not self.enable_hr:

-            x = create_random_tensors([opt_C, self.height // opt_f, self.width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)

-            

+    def create_dummy_mask(self, x):

+        if self.sampler.conditioning_key in {'hybrid', 'concat'}:

             # The "masked-image" in this case will just be all zeros since the entire image is masked.

             image_conditioning = torch.zeros(x.shape[0], 3, self.height, self.width, device=x.device)

             image_conditioning = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(image_conditioning)) 

@@ -555,11 +551,23 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
             image_conditioning = torch.nn.functional.pad(image_conditioning, (0, 0, 0, 0, 1, 0), value=1.0)

             image_conditioning = image_conditioning.to(x.dtype)

 

-            samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=image_conditioning)

+        else:

+            # Dummy zero conditioning if we're not using inpainting model.

+            # Still takes up a bit of memory, but no encoder call.

+            image_conditioning = torch.zeros(x.shape[0], 5, x.shape[-2], x.shape[-1], dtype=x.dtype, device=x.device)

+

+        return image_conditioning

+

+    def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength):

+        self.sampler = sd_samplers.create_sampler_with_index(sd_samplers.samplers, self.sampler_index, self.sd_model)

+

+        if not self.enable_hr:

+            x = create_random_tensors([opt_C, self.height // opt_f, self.width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)

+            samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.create_dummy_mask(x))

             return samples

 

         x = create_random_tensors([opt_C, self.firstphase_height // opt_f, self.firstphase_width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)

-        samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning)

+        samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.create_dummy_mask(x))

 

         samples = samples[:, :, self.truncate_y//2:samples.shape[2]-self.truncate_y//2, self.truncate_x//2:samples.shape[3]-self.truncate_x//2]

 

@@ -596,7 +604,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         x = None

         devices.torch_gc()

 

-        samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.steps)

+        samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.steps, image_conditioning=self.create_dummy_mask(samples))

 

         return samples

 

@@ -723,26 +731,36 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
             elif self.inpainting_fill == 3:

                 self.init_latent = self.init_latent * self.mask

 

-        if self.image_mask is not None:

-            conditioning_mask = np.array(self.image_mask.convert("L"))

-            conditioning_mask = conditioning_mask.astype(np.float32) / 255.0

-            conditioning_mask = torch.from_numpy(conditioning_mask[None, None])

+        conditioning_key = self.sampler.conditioning_key

 

-            # Inpainting model uses a discretized mask as input, so we round to either 1.0 or 0.0

-            conditioning_mask = torch.round(conditioning_mask)

+        if conditioning_key in {'hybrid', 'concat'}:

+            if self.image_mask is not None:

+                conditioning_mask = np.array(self.image_mask.convert("L"))

+                conditioning_mask = conditioning_mask.astype(np.float32) / 255.0

+                conditioning_mask = torch.from_numpy(conditioning_mask[None, None])

+

+                # Inpainting model uses a discretized mask as input, so we round to either 1.0 or 0.0

+                conditioning_mask = torch.round(conditioning_mask)

+            else:

+                conditioning_mask = torch.ones(1, 1, *image.shape[-2:])

+

+            # Create another latent image, this time with a masked version of the original input.

+            conditioning_mask = conditioning_mask.to(image.device)

+            conditioning_image = image * (1.0 - conditioning_mask)

+            conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(conditioning_image))

+

+            # Create the concatenated conditioning tensor to be fed to `c_concat`

+            conditioning_mask = torch.nn.functional.interpolate(conditioning_mask, size=self.init_latent.shape[-2:])

+            conditioning_mask = conditioning_mask.expand(conditioning_image.shape[0], -1, -1, -1)

+            self.image_conditioning = torch.cat([conditioning_mask, conditioning_image], dim=1)

+            self.image_conditioning = self.image_conditioning.to(shared.device).type(self.sd_model.dtype)

         else:

-            conditioning_mask = torch.ones(1, 1, *image.shape[-2:])

-

-        # Create another latent image, this time with a masked version of the original input.

-        conditioning_mask = conditioning_mask.to(image.device)

-        conditioning_image = image * (1.0 - conditioning_mask)

-        conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(conditioning_image))

-

-        # Create the concatenated conditioning tensor to be fed to `c_concat`

-        conditioning_mask = torch.nn.functional.interpolate(conditioning_mask, size=self.init_latent.shape[-2:])

-        conditioning_mask = conditioning_mask.expand(conditioning_image.shape[0], -1, -1, -1)

-        self.image_conditioning = torch.cat([conditioning_mask, conditioning_image], dim=1)

-        self.image_conditioning = self.image_conditioning.to(shared.device).type(self.sd_model.dtype)

+            self.image_conditioning = torch.zeros(

+                self.init_latent.shape[0], 5, self.init_latent.shape[-2], self.init_latent.shape[-1], 

+                dtype=self.init_latent.dtype, 

+                device=self.init_latent.device

+            )

+

 

     def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength):

         x = create_random_tensors([opt_C, self.height // opt_f, self.width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)