1// 2// Copyright 2019 Ettus Research, A National Instruments Company 3// 4// SPDX-License-Identifier: LGPL-3.0-or-later 5// 6// Module: PkgTestExec 7// 8// Description: This package provides infrastructure for tracking the state of 9// testbench execution and the results of each test. 10// 11 12 13 14package PkgTestExec; 15 16 timeunit 1ns; 17 timeprecision 1ps; 18 19 typedef enum { SEV_INFO, SEV_WARNING, SEV_ERROR, SEV_FATAL } severity_t; 20 21 typedef std::process timeout_t; 22 23 class TestExec; 24 25 string tb_name; // Name of the testbench 26 string current_test; // Name of the current test being run 27 int num_started, num_finished; // Number of tests started and finished 28 int num_passed; // Number of tests that have passed 29 int num_assertions; // Number of assertions checked for the current test 30 time start_time, end_time; // Start and end time of the testbench 31 bit stop_on_error = 1; // Configuration option to stop when an error occurs 32 bit done = 0; // Flag that sets when tb is finished 33 bit test_status[$]; // Pass/fail status of each test 34 35 timeout_t tb_timeout; // Handle to timeout for the overall testbench 36 timeout_t test_timeout; // Handle to timeout for current test 37 38 semaphore test_sem; // Testbench semaphore 39 40 41 function new(); 42 $timeformat(-9, 0, " ns", 12); 43 this.test_sem = new(1); 44 endfunction : new 45 46 47 // Call start_tb() at the start of a testbench to initialize state tracking 48 // properties. 49 // 50 // time_limit: Max simulation time allowed before at timeout occurs. 51 // This is a catch-all, in case things go very wrong during 52 // simulation and cause it to never end. 53 // 54 task start_tb(string tb_name, realtime time_limit = 10ms); 55 // Get the sempahore, to prevent multiple overlapping instances of the 56 // same testbench. 57 test_sem.get(); 58 done = 0; 59 $display("========================================================"); 60 $display("TESTBENCH STARTED: %s", tb_name); 61 $display("========================================================"); 62 this.tb_name = tb_name; 63 start_time = $time; 64 test_status = {}; 65 num_started = 0; 66 num_finished = 0; 67 num_passed = 0; 68 start_timeout( 69 tb_timeout, 70 time_limit, 71 $sformatf("Testbench \"%s\" time limit exceeded", tb_name), 72 SEV_FATAL 73 ); 74 endtask : start_tb 75 76 77 // Call end_tb() at the end of a testbench to report final statistics and, 78 // optionally, end simulation. 79 // 80 // finish: Set to 1 (default) to cause $finish() to be called at the 81 // end of simulation, cuasing the simulator to close. 82 // 83 function void end_tb(bit finish = 1); 84 assert (num_started == num_finished) else begin 85 $fatal(1, "Testbench ended before test completed"); 86 end 87 end_time = $time; 88 $display("========================================================"); 89 $display("TESTBENCH FINISHED: %s", tb_name); 90 $display(" - Time elapsed: %0t", end_time - start_time); 91 $display(" - Tests Run: %0d", num_finished); 92 $display(" - Tests Passed: %0d", num_passed); 93 $display(" - Tests Failed: %0d", num_started - num_passed); 94 $display("Result: %s", 95 (num_started == num_passed) && (num_finished > 0) 96 ? "PASSED " : "FAILED!!!"); 97 $display("========================================================"); 98 99 end_timeout(tb_timeout); 100 101 done = 1; 102 if (finish) $finish(); 103 104 // Release the semaphore to allow new instances of the testbench to run 105 test_sem.put(); 106 endfunction : end_tb 107 108 109 // Call at the start of each test with the name of the test. 110 // 111 // test_name: String name for the test to be started 112 // 113 task start_test(string test_name, realtime time_limit = 0); 114 // Make sure a there isn't already a test running 115 assert (num_started == num_finished) else begin 116 $fatal(1, "Test started before completing previous test"); 117 end 118 119 // Create a timeout for this test 120 if (time_limit > 0) begin 121 start_timeout( 122 test_timeout, 123 time_limit, 124 $sformatf("Test \"%s\" time limit exceeded", test_name), 125 SEV_FATAL 126 ); 127 end 128 129 current_test = test_name; 130 num_started++; 131 $display("[TEST CASE %3d] (t = %t) BEGIN: %s...", num_started, $time, test_name); 132 test_status.push_back(1); // Set status to 1 (passed) initially 133 num_assertions = 0; 134 endtask : start_test 135 136 137 // Call end_test() at the end of each test. 138 // 139 // test_result: Optional value to indicate the overall pass/fail result 140 // of the test. Use non-zero for pass, 0 for fail. 141 // 142 task end_test(int test_result = 1); 143 bit passed; 144 145 assert (num_started == num_finished + 1) else begin 146 $fatal(1, "No test running"); 147 end 148 149 end_timeout(test_timeout); 150 151 passed = test_status[num_started-1] && test_result; 152 num_finished++; 153 $display("[TEST CASE %3d] (t = %t) DONE... %s", 154 num_started, $time, passed ? "Passed" : "FAILED"); 155 156 if (passed) num_passed++; 157 158 current_test = ""; 159 endtask : end_test 160 161 162 // Assert the given expression and call $error() if it fails. Simulation 163 // will also be stopped (using $stop) if stop_on_error is true. 164 // 165 // expr: The expression value to be asserted 166 // message: String to report if the assertion fails 167 // 168 function void assert_error(int expr, string message = ""); 169 num_assertions++; 170 assert (expr) else begin 171 test_status[num_started] = 0; 172 $error(message); 173 if (stop_on_error) $stop; 174 end 175 endfunction : assert_error 176 177 178 // Assert the given expression and call $fatal() if it fails. 179 // 180 // expr: The expression value to be asserted 181 // message: String to report if the assertion fails 182 // 183 function void assert_fatal(int expr, string message = ""); 184 num_assertions++; 185 assert (expr) else begin 186 test_status[num_started] = 0; 187 $fatal(1, message); 188 end 189 endfunction : assert_fatal 190 191 192 // Assert the given expression and call $warning() if it fails. 193 // 194 // expr: The expression value to be asserted 195 // message: String to report if the assertion fails 196 // 197 function void assert_warning(int expr, string message = ""); 198 num_assertions++; 199 assert (expr) else begin 200 $warning(message); 201 end 202 endfunction : assert_warning 203 204 205 // Assert the given expression and call the appropriate severity or fatal 206 // task if the expression fails. 207 // 208 // expr: The expression value to be asserted 209 // message: String to report if the assertion fails 210 // severity: Indicates the type of severity task that should be used if 211 // the assertion fails ($info, $warning, $error, $fatal). 212 // Default value is SEV_ERROR. 213 // 214 function void assert_sev( 215 int expr, 216 string message = "", 217 severity_t severity = SEV_ERROR 218 ); 219 num_assertions++; 220 assert (expr) else begin 221 case (severity) 222 SEV_INFO: begin 223 $info(message); 224 end 225 SEV_WARNING: begin 226 $warning(message); 227 end 228 SEV_ERROR: begin 229 $error(message); 230 test_status[num_started] = 0; 231 if (stop_on_error) $stop; 232 end 233 default: begin // SEV_FATAL 234 test_status[num_started] = 0; 235 $fatal(1, message); 236 end 237 endcase 238 end 239 endfunction : assert_sev 240 241 242 // Create a timeout that will expire after the indicated delay, causing an 243 // error if the timeout is not canceled before the delay has elapsed. 244 // 245 // handle: A handle to the timeout process created. Use this 246 // handle to end the timeout. 247 // timeout_delay: Amount of time to wait before the timeout expires. 248 // message: String message to display if the timeout expires. 249 // severity: Indicates the type of severity task that should be 250 // used if the timeout expires. Default is SEV_ERROR. 251 // 252 task start_timeout( 253 output timeout_t handle, 254 input realtime timeout_delay, 255 input string message = "Timeout", 256 input severity_t severity = SEV_ERROR 257 ); 258 mailbox #(std::process) pid = new(1); 259 fork 260 begin : timeout 261 pid.put(process::self()); 262 #(timeout_delay); 263 assert_sev(0, {"Timeout: ", message}, severity); 264 end 265 join_none 266 #0; // Start the timeout process so we can get its process ID 267 268 // Return the process ID 269 pid.get(handle); 270 endtask 271 272 273 // Cancel the timeout with the given handle. This kills the process 274 // running the timeout delay. 275 // 276 // handle: Handle created by start_timeout() for the timeout process. 277 // 278 function void end_timeout(timeout_t handle); 279 if (handle != null) 280 if (handle.status != process::FINISHED) handle.kill(); 281 endfunction; 282 283 endclass : TestExec 284 285 286 //--------------------------------------------------------------------------- 287 // Test Object 288 //--------------------------------------------------------------------------- 289 // 290 // This is the default TestExec object instance shared by all instances of 291 // the running the testbench. 292 // 293 //--------------------------------------------------------------------------- 294 295 TestExec test = new(); 296 297endpackage : PkgTestExec 298 299