Module `neuralops.tests.test_svd_lowrank_comprehensive`

Functions

def test_adaptive_svd()

Expand source code

def test_adaptive_svd():
    config = DummyConfig()
    model = AdaptiveSVDAttention(config)
    x = torch.randn(2, 128, 768, requires_grad=True)
    out, _ = model(x)
    assert out.shape == (2, 128, 768)
    loss = out.sum()
    loss.backward()
    assert torch.isfinite(loss)
    assert x.grad is not None and x.grad.abs().sum() > 0
    print("AdaptiveSVDAttention passed!")

def test_cur()

Expand source code

def test_cur():
    model = nn.Sequential(nn.Linear(768, 768), nn.ReLU(), nn.Linear(768, 768))
    config = DummyConfig()
    compressed = cur_compressor_factory(model, comp_config=config, rank=64)
    x = torch.randn(2, 128, 768, requires_grad=True)
    out = compressed(x)
    assert out.shape == (2, 128, 768)
    loss = out.sum()
    loss.backward()
    assert torch.isfinite(loss)
    assert x.grad is not None and x.grad.abs().sum() > 0
    print("CUR passed!")

def test_kronecker_tt_lora()

Expand source code

def test_kronecker_tt_lora():
    model = nn.Sequential(nn.Linear(768, 768), nn.ReLU(), nn.Linear(768, 768))
    config = DummyConfig()
    compressed = kronecker_tt_lora_factory(model, comp_config=config)
    x = torch.randn(2, 128, 768, requires_grad=True)
    out = compressed(x)
    assert out.shape == (2, 128, 768)
    loss = out.sum()
    loss.backward()
    assert torch.isfinite(loss)
    assert x.grad is not None and x.grad.abs().sum() > 0
    print("KroneckerTTLoRA passed!")

Classes

class DummyConfig

Expand source code

class DummyConfig:
    num_attention_heads = 12
    hidden_size = 768
    num_heads = 12
    d_model = 768
    svd_rank = 32
    residual_rank = 8
    dropout = 0.1
    energy_threshold = 0.92

Class variables

var d_model: The type of the None singleton.
var dropout: The type of the None singleton.
var energy_threshold: The type of the None singleton.
var hidden_size: The type of the None singleton.
var num_attention_heads: The type of the None singleton.
var num_heads: The type of the None singleton.
var residual_rank: The type of the None singleton.
var svd_rank: The type of the None singleton.