Merge branch 'AUTOMATIC1111:master' into interrogate

1 files changed, 12 insertions, 7 deletions

diff --git a/extensions-builtin/Lora/lora.py b/extensions-builtin/Lora/lora.py
index da1797dc..137e58f7 100644
--- a/extensions-builtin/Lora/lora.py
+++ b/extensions-builtin/Lora/lora.py
@@ -57,6 +57,7 @@ class LoraUpDownModule:
     def __init__(self):

         self.up = None

         self.down = None

+        self.alpha = None

 

 

 def assign_lora_names_to_compvis_modules(sd_model):

@@ -92,6 +93,15 @@ def load_lora(name, filename):
             keys_failed_to_match.append(key_diffusers)

             continue

 

+        lora_module = lora.modules.get(key, None)

+        if lora_module is None:

+            lora_module = LoraUpDownModule()

+            lora.modules[key] = lora_module

+

+        if lora_key == "alpha":

+            lora_module.alpha = weight.item()

+            continue

+

         if type(sd_module) == torch.nn.Linear:

             module = torch.nn.Linear(weight.shape[1], weight.shape[0], bias=False)

         elif type(sd_module) == torch.nn.Conv2d:

@@ -104,17 +114,12 @@ def load_lora(name, filename):
 

         module.to(device=devices.device, dtype=devices.dtype)

 

-        lora_module = lora.modules.get(key, None)

-        if lora_module is None:

-            lora_module = LoraUpDownModule()

-            lora.modules[key] = lora_module

-

         if lora_key == "lora_up.weight":

             lora_module.up = module

         elif lora_key == "lora_down.weight":

             lora_module.down = module

         else:

-            assert False, f'Bad Lora layer name: {key_diffusers} - must end in lora_up.weight or lora_down.weight'

+            assert False, f'Bad Lora layer name: {key_diffusers} - must end in lora_up.weight, lora_down.weight or alpha'

 

     if len(keys_failed_to_match) > 0:

         print(f"Failed to match keys when loading Lora {filename}: {keys_failed_to_match}")

@@ -161,7 +166,7 @@ def lora_forward(module, input, res):
     for lora in loaded_loras:

         module = lora.modules.get(lora_layer_name, None)

         if module is not None:

-            res = res + module.up(module.down(input)) * lora.multiplier

+            res = res + module.up(module.down(input)) * lora.multiplier * (module.alpha / module.up.weight.shape[1] if module.alpha else 1.0)

 

     return res