Merge pull request #7309 from brkirch/fix-embeddings

Fix embeddings, upscalers, and refactor `--upcast-sampling`

1 files changed, 8 insertions, 7 deletions

diff --git a/modules/processing.py b/modules/processing.py
index 5072fc40..afab6790 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -173,8 +173,7 @@ class StableDiffusionProcessing:
         midas_in = torch.from_numpy(transformed["midas_in"][None, ...]).to(device=shared.device)

         midas_in = repeat(midas_in, "1 ... -> n ...", n=self.batch_size)

 

-        conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(source_image.to(devices.dtype_vae) if devices.unet_needs_upcast else source_image))

-        conditioning_image = conditioning_image.float() if devices.unet_needs_upcast else conditioning_image

+        conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(source_image))

         conditioning = torch.nn.functional.interpolate(

             self.sd_model.depth_model(midas_in),

             size=conditioning_image.shape[2:],

@@ -218,7 +217,7 @@ class StableDiffusionProcessing:
         )

 

         # Encode the new masked image using first stage of network.

-        conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(conditioning_image.to(devices.dtype_vae) if devices.unet_needs_upcast else conditioning_image))

+        conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(conditioning_image))

 

         # Create the concatenated conditioning tensor to be fed to `c_concat`

         conditioning_mask = torch.nn.functional.interpolate(conditioning_mask, size=latent_image.shape[-2:])

@@ -229,16 +228,18 @@ class StableDiffusionProcessing:
         return image_conditioning

 

     def img2img_image_conditioning(self, source_image, latent_image, image_mask=None):

+        source_image = devices.cond_cast_float(source_image)

+

         # HACK: Using introspection as the Depth2Image model doesn't appear to uniquely

         # identify itself with a field common to all models. The conditioning_key is also hybrid.

         if isinstance(self.sd_model, LatentDepth2ImageDiffusion):

-            return self.depth2img_image_conditioning(source_image.float() if devices.unet_needs_upcast else source_image)

+            return self.depth2img_image_conditioning(source_image)

 

         if self.sd_model.cond_stage_key == "edit":

             return self.edit_image_conditioning(source_image)

 

         if self.sampler.conditioning_key in {'hybrid', 'concat'}:

-            return self.inpainting_image_conditioning(source_image.float() if devices.unet_needs_upcast else source_image, latent_image, image_mask=image_mask)

+            return self.inpainting_image_conditioning(source_image, latent_image, image_mask=image_mask)

 

         # Dummy zero conditioning if we're not using inpainting or depth model.

         return latent_image.new_zeros(latent_image.shape[0], 5, 1, 1)

@@ -418,7 +419,7 @@ def create_random_tensors(shape, seeds, subseeds=None, subseed_strength=0.0, see
 

 def decode_first_stage(model, x):

     with devices.autocast(disable=x.dtype == devices.dtype_vae):

-        x = model.decode_first_stage(x.to(devices.dtype_vae) if devices.unet_needs_upcast else x)

+        x = model.decode_first_stage(x)

 

     return x

 

@@ -1007,7 +1008,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
 

         image = torch.from_numpy(batch_images)

         image = 2. * image - 1.

-        image = image.to(device=shared.device, dtype=devices.dtype_vae if devices.unet_needs_upcast else None)

+        image = image.to(shared.device)

 

         self.init_latent = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(image))