Enables comprehensive benchmark test configurations

Uncomments and activates previously disabled benchmark test cases to enable comprehensive performance testing across various parameter configurations.

Includes inference tests with different sequence lengths, batch size variations, head count and dimension testing, window size experiments, and non-causal attention benchmarks.

Also fixes inference test parameter from 2 to 1 for proper single-token inference evaluation.

Author: LoserCheems

benchmarks/forward_performance.py

@@ -732,57 +732,57 @@ def run_performance_benchmark(test_type='all', num_runs=3, warmup_runs=2):
         (1, 2, 1, 4096, 4096, 128, 2048, True),
         (1, 2, 1, 8192, 8192, 128, 2048, True),
         (1, 2, 1, 16384, 16384, 128, 2048, True),
-        # (1, 2, 1, 32768, 32768, 128, 2048, True),
+        (1, 2, 1, 32768, 32768, 128, 2048, True),
 
         # # Inference
-        # (1, 2, 1, 2, 256, 128, 2048, True),
-        # (1, 2, 1, 2, 512, 128, 2048, True),
-        # (1, 2, 1, 2, 1024, 128, 2048, True),
-        # (1, 2, 1, 2, 2048, 128, 2048, True),
-        # (1, 2, 1, 2, 4096, 128, 2048, True),
-        # (1, 2, 1, 2, 8192, 128, 2048, True),
-        # (1, 2, 1, 2, 16384, 128, 2048, True),
-        # (1, 2, 1, 2, 32768, 128, 2048, True),
-        (1, 2, 1, 2, 65536, 128, 2048, True),
-        # (1, 2, 1, 2, 131072, 128, 2048, True),
-        # (1, 2, 1, 2, 262144, 128, 2048, True),
-        # (1, 2, 1, 2, 524288, 128, 2048, True),
-        
-        # # Vary batch size
-        # (1, 2, 1, 4096, 4096, 32, 2048, True),
-        # (2, 2, 1, 4096, 4096, 32, 2048, True),
-        # (4, 2, 1, 4096, 4096, 32, 2048, True),
-        # (8, 2, 1, 4096, 4096, 32, 2048, True),
-        
-        # # Vary head count
-        # (1, 1, 1, 4096, 4096, 32, 2048, True),
-        # (1, 2, 1, 4096, 4096, 32, 2048, True),
-        # (1, 4, 1, 4096, 4096, 32, 2048, True),
-        # (1, 8, 2, 4096, 4096, 32, 2048, True),
-        
-        # # Vary head dimension
-        # (1, 2, 1, 4096, 4096, 32, 2048, True),
-        # (1, 2, 1, 4096, 4096, 64, 2048, True),
-        # (1, 2, 1, 4096, 4096, 96, 2048, True),
-        # (1, 2, 1, 4096, 4096, 128, 2048, True),
-        # (1, 2, 1, 4096, 4096, 192, 2048, True),
-        # (1, 2, 1, 4096, 4096, 256, 2048, True),
-        
-        # # Vary keep_window_size
-        # (1, 2, 1, 32768, 32768, 128, 32, True),
-        # (1, 2, 1, 32768, 32768, 128, 64, True),
-        # (1, 2, 1, 32768, 32768, 128, 128, True),
-        # (1, 2, 1, 32768, 32768, 128, 256, True),
-        # (1, 2, 1, 32768, 32768, 128, 512, True),
-        # (1, 2, 1, 32768, 32768, 128, 1024, True),
-        # (1, 2, 1, 32768, 32768, 128, 2048, True),
-        # (1, 2, 1, 32768, 32768, 128, 4096, True),
-        # (1, 2, 1, 32768, 32768, 128, 8192, True),
-        # (1, 2, 1, 32768, 32768, 128, 16384, True),
-        # (1, 2, 1, 32768, 32768, 128, 32768, True),
-        
-        # # Test non-causal
-        # (1, 2, 1, 4096, 4096, 128, 2048, False),
+        (1, 2, 1, 1, 256, 128, 2048, True),
+        (1, 2, 1, 1, 512, 128, 2048, True),
+        (1, 2, 1, 1, 1024, 128, 2048, True),
+        (1, 2, 1, 1, 2048, 128, 2048, True),
+        (1, 2, 1, 1, 4096, 128, 2048, True),
+        (1, 2, 1, 1, 8192, 128, 2048, True),
+        (1, 2, 1, 1, 16384, 128, 2048, True),
+        (1, 2, 1, 1, 32768, 128, 2048, True),
+        (1, 2, 1, 1, 65536, 128, 2048, True),
+        (1, 2, 1, 1, 131072, 128, 2048, True),
+        (1, 2, 1, 1, 262144, 128, 2048, True),
+        (1, 2, 1, 1, 524288, 128, 2048, True),
+        
+        # Vary batch size
+        (1, 2, 1, 4096, 4096, 32, 2048, True),
+        (2, 2, 1, 4096, 4096, 32, 2048, True),
+        (4, 2, 1, 4096, 4096, 32, 2048, True),
+        (8, 2, 1, 4096, 4096, 32, 2048, True),
+        
+        # Vary head count
+        (1, 1, 1, 4096, 4096, 32, 2048, True),
+        (1, 2, 1, 4096, 4096, 32, 2048, True),
+        (1, 4, 1, 4096, 4096, 32, 2048, True),
+        (1, 8, 2, 4096, 4096, 32, 2048, True),
+        
+        # Vary head dimension
+        (1, 2, 1, 4096, 4096, 32, 2048, True),
+        (1, 2, 1, 4096, 4096, 64, 2048, True),
+        (1, 2, 1, 4096, 4096, 96, 2048, True),
+        (1, 2, 1, 4096, 4096, 128, 2048, True),
+        (1, 2, 1, 4096, 4096, 192, 2048, True),
+        (1, 2, 1, 4096, 4096, 256, 2048, True),
+        
+        # Vary keep_window_size
+        (1, 2, 1, 32768, 32768, 128, 32, True),
+        (1, 2, 1, 32768, 32768, 128, 64, True),
+        (1, 2, 1, 32768, 32768, 128, 128, True),
+        (1, 2, 1, 32768, 32768, 128, 256, True),
+        (1, 2, 1, 32768, 32768, 128, 512, True),
+        (1, 2, 1, 32768, 32768, 128, 1024, True),
+        (1, 2, 1, 32768, 32768, 128, 2048, True),
+        (1, 2, 1, 32768, 32768, 128, 4096, True),
+        (1, 2, 1, 32768, 32768, 128, 8192, True),
+        (1, 2, 1, 32768, 32768, 128, 16384, True),
+        (1, 2, 1, 32768, 32768, 128, 32768, True),
+        
+        # Test non-causal
+        (1, 2, 1, 4096, 4096, 128, 2048, False),
     ]
 
     print(f"\n📊 Benchmark Results (averaged over {num_runs} runs):")