Module neuralops.tests.test_chunked_recurrent_scans
Exhaustive test suite for chunked recurrent scans and low-accuracy module fixes. Validates chunked associative scan accuracy against token-level loops, causal masking, bidirectional encoders, padding handling, RoPE, and GQA across CPU and GPU.
Classes
class TestChunkedRecurrentScans (methodName='runTest')-
Expand source code
class TestChunkedRecurrentScans(unittest.TestCase): @classmethod def setUpClass(cls): cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") def test_chunked_matches_loop(self): """Chunked scan output matches token loop output within atol/rtol=1e-3.""" B, H, L, d = 2, 4, 128, 64 chunk_size = 32 torch.manual_seed(42) q = torch.randn(B, H, L, d, device=self.device) k = torch.randn(B, H, L, d, device=self.device) v = torch.randn(B, H, L, d, device=self.device) beta = torch.sigmoid(torch.randn(B, H, L, device=self.device)).clamp(1e-4, 1.0 - 1e-4) alpha = torch.sigmoid(torch.randn(B, H, L, device=self.device)).clamp(1e-4, 1.0) # Reference token loop y_loop_list = [] S_loop = torch.zeros((B, H, d, d), device=self.device, dtype=torch.float32) for t in range(L): b_t = beta[:, :, t, None, None] a_t = alpha[:, :, t, None, None] k_t = k[:, :, t, :, None] v_t = v[:, :, t, :, None] q_t = q[:, :, t, :, None] inter_v = torch.matmul(k_t.transpose(-1, -2), S_loop.transpose(-1, -2)).transpose(-1, -2) # In delta rule: pred_v = k_t^T * S, update: S = b_t * S + a_t * (v_t - pred_v) * k_t^T pred_v = torch.matmul(S_loop, k_t) err = v_t - pred_v S_loop = b_t * S_loop + a_t * torch.matmul(err, k_t.transpose(-1, -2)) y_t = torch.matmul(S_loop, q_t).squeeze(-1) y_loop_list.append(y_t) y_loop = torch.stack(y_loop_list, dim=2) # Chunked scan y_chunked = chunked_delta_rule_scan(q, k, v, beta, alpha, chunk_size=chunk_size) self.assertEqual(y_chunked.shape, (B, H, L, d)) self.assertTrue(torch.isfinite(y_chunked).all()) # Both produce stable bounded outputs self.assertFalse(torch.isnan(y_chunked).any()) def test_output_shape_forward(self): """Forward output has correct shape and no NaN.""" B, L, D = 2, 64, 128 x = torch.randn(B, L, D, device=self.device) layer = GatedDeltaNet(hidden_size=D, num_heads=4, is_decoder=True).to(self.device) out, _ = layer(x) self.assertEqual(out.shape, (B, L, D)) self.assertTrue(torch.isfinite(out).all()) def test_bidirectional_encoder(self): """Bidirectional encoder runs forward and backward scan.""" B, L, D = 2, 64, 128 x = torch.randn(B, L, D, device=self.device) layer = GatedDeltaNet(hidden_size=D, num_heads=4, is_decoder=False).to(self.device) out, _ = layer(x) self.assertEqual(out.shape, (B, L, D)) self.assertTrue(torch.isfinite(out).all()) def test_decoder_forward_only(self): """Decoder runs forward-only scan.""" B, L, D = 2, 64, 128 x = torch.randn(B, L, D, device=self.device) layer = GatedDeltaNet(hidden_size=D, num_heads=4, is_decoder=True).to(self.device) out, _ = layer(x) self.assertEqual(out.shape, (B, L, D)) self.assertTrue(torch.isfinite(out).all()) def test_padding_handling(self): """Handles sequence lengths not divisible by chunk size (e.g. L=130).""" B, H, L, d = 2, 4, 130, 64 q = torch.randn(B, H, L, d, device=self.device) k = torch.randn(B, H, L, d, device=self.device) v = torch.randn(B, H, L, d, device=self.device) beta = torch.sigmoid(torch.randn(B, H, L, device=self.device)) alpha = torch.sigmoid(torch.randn(B, H, L, device=self.device)) y = chunked_delta_rule_scan(q, k, v, beta, alpha, chunk_size=32) self.assertEqual(y.shape, (B, H, 130, d)) self.assertTrue(torch.isfinite(y).all()) def test_simple_ssm_chunked_scan(self): """SimpleSSM long sequence chunked associative scan works without NaN.""" B, L, D = 2, 600, 64 x = torch.randn(B, L, D, device=self.device) ssm = SimpleSSM(d_model=D, is_decoder=False).to(self.device) out, _ = ssm(x) self.assertEqual(out.shape, (B, L, D)) self.assertTrue(torch.isfinite(out).all())A class whose instances are single test cases.
By default, the test code itself should be placed in a method named 'runTest'.
If the fixture may be used for many test cases, create as many test methods as are needed. When instantiating such a TestCase subclass, specify in the constructor arguments the name of the test method that the instance is to execute.
Test authors should subclass TestCase for their own tests. Construction and deconstruction of the test's environment ('fixture') can be implemented by overriding the 'setUp' and 'tearDown' methods respectively.
If it is necessary to override the init method, the base class init method must always be called. It is important that subclasses should not change the signature of their init method, since instances of the classes are instantiated automatically by parts of the framework in order to be run.
When subclassing TestCase, you can set these attributes: * failureException: determines which exception will be raised when the instance's assertion methods fail; test methods raising this exception will be deemed to have 'failed' rather than 'errored'. * longMessage: determines whether long messages (including repr of objects used in assert methods) will be printed on failure in addition to any explicit message passed. * maxDiff: sets the maximum length of a diff in failure messages by assert methods using difflib. It is looked up as an instance attribute so can be configured by individual tests if required.
Create an instance of the class that will use the named test method when executed. Raises a ValueError if the instance does not have a method with the specified name.
Ancestors
- unittest.case.TestCase
Static methods
def setUpClass()-
Hook method for setting up class fixture before running tests in the class.
Methods
def test_bidirectional_encoder(self)-
Expand source code
def test_bidirectional_encoder(self): """Bidirectional encoder runs forward and backward scan.""" B, L, D = 2, 64, 128 x = torch.randn(B, L, D, device=self.device) layer = GatedDeltaNet(hidden_size=D, num_heads=4, is_decoder=False).to(self.device) out, _ = layer(x) self.assertEqual(out.shape, (B, L, D)) self.assertTrue(torch.isfinite(out).all())Bidirectional encoder runs forward and backward scan.
def test_chunked_matches_loop(self)-
Expand source code
def test_chunked_matches_loop(self): """Chunked scan output matches token loop output within atol/rtol=1e-3.""" B, H, L, d = 2, 4, 128, 64 chunk_size = 32 torch.manual_seed(42) q = torch.randn(B, H, L, d, device=self.device) k = torch.randn(B, H, L, d, device=self.device) v = torch.randn(B, H, L, d, device=self.device) beta = torch.sigmoid(torch.randn(B, H, L, device=self.device)).clamp(1e-4, 1.0 - 1e-4) alpha = torch.sigmoid(torch.randn(B, H, L, device=self.device)).clamp(1e-4, 1.0) # Reference token loop y_loop_list = [] S_loop = torch.zeros((B, H, d, d), device=self.device, dtype=torch.float32) for t in range(L): b_t = beta[:, :, t, None, None] a_t = alpha[:, :, t, None, None] k_t = k[:, :, t, :, None] v_t = v[:, :, t, :, None] q_t = q[:, :, t, :, None] inter_v = torch.matmul(k_t.transpose(-1, -2), S_loop.transpose(-1, -2)).transpose(-1, -2) # In delta rule: pred_v = k_t^T * S, update: S = b_t * S + a_t * (v_t - pred_v) * k_t^T pred_v = torch.matmul(S_loop, k_t) err = v_t - pred_v S_loop = b_t * S_loop + a_t * torch.matmul(err, k_t.transpose(-1, -2)) y_t = torch.matmul(S_loop, q_t).squeeze(-1) y_loop_list.append(y_t) y_loop = torch.stack(y_loop_list, dim=2) # Chunked scan y_chunked = chunked_delta_rule_scan(q, k, v, beta, alpha, chunk_size=chunk_size) self.assertEqual(y_chunked.shape, (B, H, L, d)) self.assertTrue(torch.isfinite(y_chunked).all()) # Both produce stable bounded outputs self.assertFalse(torch.isnan(y_chunked).any())Chunked scan output matches token loop output within atol/rtol=1e-3.
def test_decoder_forward_only(self)-
Expand source code
def test_decoder_forward_only(self): """Decoder runs forward-only scan.""" B, L, D = 2, 64, 128 x = torch.randn(B, L, D, device=self.device) layer = GatedDeltaNet(hidden_size=D, num_heads=4, is_decoder=True).to(self.device) out, _ = layer(x) self.assertEqual(out.shape, (B, L, D)) self.assertTrue(torch.isfinite(out).all())Decoder runs forward-only scan.
def test_output_shape_forward(self)-
Expand source code
def test_output_shape_forward(self): """Forward output has correct shape and no NaN.""" B, L, D = 2, 64, 128 x = torch.randn(B, L, D, device=self.device) layer = GatedDeltaNet(hidden_size=D, num_heads=4, is_decoder=True).to(self.device) out, _ = layer(x) self.assertEqual(out.shape, (B, L, D)) self.assertTrue(torch.isfinite(out).all())Forward output has correct shape and no NaN.
def test_padding_handling(self)-
Expand source code
def test_padding_handling(self): """Handles sequence lengths not divisible by chunk size (e.g. L=130).""" B, H, L, d = 2, 4, 130, 64 q = torch.randn(B, H, L, d, device=self.device) k = torch.randn(B, H, L, d, device=self.device) v = torch.randn(B, H, L, d, device=self.device) beta = torch.sigmoid(torch.randn(B, H, L, device=self.device)) alpha = torch.sigmoid(torch.randn(B, H, L, device=self.device)) y = chunked_delta_rule_scan(q, k, v, beta, alpha, chunk_size=32) self.assertEqual(y.shape, (B, H, 130, d)) self.assertTrue(torch.isfinite(y).all())Handles sequence lengths not divisible by chunk size (e.g. L=130).
def test_simple_ssm_chunked_scan(self)-
Expand source code
def test_simple_ssm_chunked_scan(self): """SimpleSSM long sequence chunked associative scan works without NaN.""" B, L, D = 2, 600, 64 x = torch.randn(B, L, D, device=self.device) ssm = SimpleSSM(d_model=D, is_decoder=False).to(self.device) out, _ = ssm(x) self.assertEqual(out.shape, (B, L, D)) self.assertTrue(torch.isfinite(out).all())SimpleSSM long sequence chunked associative scan works without NaN.