aboutsummaryrefslogtreecommitdiff
path: root/extensions-builtin/Lora/lyco_helpers.py
blob: 9ea499fbd0bfa9e0242aec84a4ff94d169f9990a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
import torch


def make_weight_cp(t, wa, wb):
    temp = torch.einsum('i j k l, j r -> i r k l', t, wb)
    return torch.einsum('i j k l, i r -> r j k l', temp, wa)


def rebuild_conventional(up, down, shape, dyn_dim=None):
    up = up.reshape(up.size(0), -1)
    down = down.reshape(down.size(0), -1)
    if dyn_dim is not None:
        up = up[:, :dyn_dim]
        down = down[:dyn_dim, :]
    return (up @ down).reshape(shape)