1 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
2 # This script enables you running RocksDB tests by running
3 # All the tests concurrently and utilizing all the cores
4 Param(
5 [switch]$EnableJE = $false, # Look for and use test executable, append _je to listed exclusions
6 [switch]$RunAll = $false, # Will attempt discover all *_test[_je].exe binaries and run all
7 # of them as Google suites. I.e. It will run test cases concurrently
8 # except those mentioned as $Run, those will run as individual test cases
9 # And any execlued with $ExcludeExes or $ExcludeCases
10 # It will also not run any individual test cases
11 # excluded but $ExcludeCasese
12 [switch]$RunAllExe = $false, # Look for and use test exdcutables, append _je to exclusions automatically
13 # It will attempt to run them in parallel w/o breaking them up on individual
14 # test cases. Those listed with $ExcludeExes will be excluded
15 [string]$SuiteRun = "", # Split test suites in test cases and run in parallel, not compatible with $RunAll
16 [string]$Run = "", # Run specified executables in parallel but do not split to test cases
17 [string]$ExcludeCases = "", # Exclude test cases, expects a comma separated list, no spaces
18 # Takes effect when $RunAll or $SuiteRun is specified. Must have full
19 # Test cases name including a group and a parameter if any
20 [string]$ExcludeExes = "", # Exclude exes from consideration, expects a comma separated list,
21 # no spaces. Takes effect only when $RunAll is specified
22 [string]$WorkFolder = "", # Direct tests to use that folder. SSD or Ram drive are better options.
23 # Number of async tasks that would run concurrently. Recommend a number below 64.
24 # However, CPU utlization really depends on the storage media. Recommend ram based disk.
25 # a value of 1 will run everything serially
26 [int]$Concurrency = 8,
27 [int]$Limit = -1 # -1 means do not limit for test purposes
28 )
29
30 # Folders and commands must be fullpath to run assuming
31 # the current folder is at the root of the git enlistment
32 $StartDate = (Get-Date)
33 $StartDate
34
35
36 $DebugPreference = "Continue"
37
38 # These tests are not google test suites and we should guard
39 # Against running them as suites
40 $RunOnly = New-Object System.Collections.Generic.HashSet[string]
41 $RunOnly.Add("c_test") | Out-Null
42 $RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
43 $RunOnly.Add("merge_test") | Out-Null
44 $RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
45 $RunOnly.Add("backupable_db_test") | Out-Null # Disabled
46 $RunOnly.Add("timer_queue_test") | Out-Null # Not a gtest
47
48 if($RunAll -and $SuiteRun -ne "") {
49 Write-Error "$RunAll and $SuiteRun are not compatible"
50 exit 1
51 }
52
53 if($RunAllExe -and $Run -ne "") {
54 Write-Error "$RunAllExe and $Run are not compatible"
55 exit 1
56 }
57
58 # If running under Appveyor assume that root
59 [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
60 if($Appveyor -ne "") {
61 $RootFolder = $Appveyor
62 } else {
63 $RootFolder = $PSScriptRoot -replace '\\build_tools', ''
64 }
65
66 $LogFolder = -Join($RootFolder, "\db_logs\")
67 $BinariesFolder = -Join($RootFolder, "\build\Debug\")
68
69 if($WorkFolder -eq "") {
70
71 # If TEST_TMPDIR is set use it
72 [string]$var = $Env:TEST_TMPDIR
73 if($var -eq "") {
74 $WorkFolder = -Join($RootFolder, "\db_tests\")
75 $Env:TEST_TMPDIR = $WorkFolder
76 } else {
77 $WorkFolder = $var
78 }
79 } else {
80 # Override from a command line
81 $Env:TEST_TMPDIR = $WorkFolder
82 }
83
84 Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
85 Write-Output "BinariesFolder: $BinariesFolder, LogFolder: $LogFolder"
86
87 # Create test directories in the current folder
88 md -Path $WorkFolder -ErrorAction Ignore | Out-Null
89 md -Path $LogFolder -ErrorAction Ignore | Out-Null
90
91
92 $ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
93 if($ExcludeCases -ne "") {
94 Write-Host "ExcludeCases: $ExcludeCases"
95 $l = $ExcludeCases -split ' '
96 ForEach($t in $l) {
97 $ExcludeCasesSet.Add($t) | Out-Null
98 }
99 }
100
101 $ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
102 if($ExcludeExes -ne "") {
103 Write-Host "ExcludeExe: $ExcludeExes"
104 $l = $ExcludeExes -split ' '
105 ForEach($t in $l) {
106 $ExcludeExesSet.Add($t) | Out-Null
107 }
108 }
109
110
111 # Extract the names of its tests by running db_test with --gtest_list_tests.
112 # This filter removes the "#"-introduced comments, and expands to
113 # fully-qualified names by changing input like this:
114 #
115 # DBTest.
116 # Empty
117 # WriteEmptyBatch
118 # MultiThreaded/MultiThreadedDBTest.
119 # MultiThreaded/0 # GetParam() = 0
120 # MultiThreaded/1 # GetParam() = 1
121 #
122 # into this:
123 #
124 # DBTest.Empty
125 # DBTest.WriteEmptyBatch
126 # MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
127 # MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
128 #
129 # Output into the parameter in a form TestName -> Log File Name
ExtractTestCases([string] $GTestExe, $HashTable)130 function ExtractTestCases([string]$GTestExe, $HashTable) {
131
132 $Tests = @()
133 # Run db_test to get a list of tests and store it into $a array
134 &$GTestExe --gtest_list_tests | tee -Variable Tests | Out-Null
135
136 # Current group
137 $Group=""
138
139 ForEach( $l in $Tests) {
140
141 # Leading whitespace is fine
142 $l = $l -replace '^\s+',''
143 # Trailing dot is a test group but no whitespace
144 if ($l -match "\.$" -and $l -notmatch "\s+") {
145 $Group = $l
146 } else {
147 # Otherwise it is a test name, remove leading space
148 $test = $l
149 # remove trailing comment if any and create a log name
150 $test = $test -replace '\s+\#.*',''
151 $test = "$Group$test"
152
153 if($ExcludeCasesSet.Contains($test)) {
154 Write-Warning "$test case is excluded"
155 continue
156 }
157
158 $test_log = $test -replace '[\./]','_'
159 $test_log += ".log"
160 $log_path = -join ($LogFolder, $test_log)
161
162 # Add to a hashtable
163 $HashTable.Add($test, $log_path);
164 }
165 }
166 }
167
168 # The function removes trailing .exe siffix if any,
169 # creates a name for the log file
170 # Then adds the test name if it was not excluded into
171 # a HashTable in a form of test_name -> log_path
MakeAndAdd([string] $token, $HashTable)172 function MakeAndAdd([string]$token, $HashTable) {
173
174 $test_name = $token -replace '.exe$', ''
175 $log_name = -join ($test_name, ".log")
176 $log_path = -join ($LogFolder, $log_name)
177 $HashTable.Add($test_name, $log_path)
178 }
179
180 # This function takes a list of Suites to run
181 # Lists all the test cases in each of the suite
182 # and populates HashOfHashes
183 # Ordered by suite(exe) @{ Exe = @{ TestCase = LogName }}
ProcessSuites($ListOfSuites, $HashOfHashes)184 function ProcessSuites($ListOfSuites, $HashOfHashes) {
185
186 $suite_list = $ListOfSuites
187 # Problem: if you run --gtest_list_tests on
188 # a non Google Test executable then it will start executing
189 # and we will get nowhere
190 ForEach($suite in $suite_list) {
191
192 if($RunOnly.Contains($suite)) {
193 Write-Warning "$suite is excluded from running as Google test suite"
194 continue
195 }
196
197 if($EnableJE) {
198 $suite += "_je"
199 }
200
201 $Cases = [ordered]@{}
202 $Cases.Clear()
203 $suite_exe = -Join ($BinariesFolder, $suite)
204 ExtractTestCases -GTestExe $suite_exe -HashTable $Cases
205 if($Cases.Count -gt 0) {
206 $HashOfHashes.Add($suite, $Cases);
207 }
208 }
209
210 # Make logs and run
211 if($CasesToRun.Count -lt 1) {
212 Write-Error "Failed to extract tests from $SuiteRun"
213 exit 1
214 }
215
216 }
217
218 # This will contain all test executables to run
219
220 # Hash table that contains all non suite
221 # Test executable to run
222 $TestExes = [ordered]@{}
223
224 # Check for test exe that are not
225 # Google Test Suites
226 # Since this is explicitely mentioned it is not subject
227 # for exclusions
228 if($Run -ne "") {
229
230 $test_list = $Run -split ' '
231 ForEach($t in $test_list) {
232
233 if($EnableJE) {
234 $t += "_je"
235 }
236 MakeAndAdd -token $t -HashTable $TestExes
237 }
238
239 if($TestExes.Count -lt 1) {
240 Write-Error "Failed to extract tests from $Run"
241 exit 1
242 }
243 } elseif($RunAllExe) {
244 # Discover all the test binaries
245 if($EnableJE) {
246 $pattern = "*_test_je.exe"
247 } else {
248 $pattern = "*_test.exe"
249 }
250
251 $search_path = -join ($BinariesFolder, $pattern)
252 Write-Host "Binaries Search Path: $search_path"
253
254 $DiscoveredExe = @()
255 dir -Path $search_path | ForEach-Object {
256 $DiscoveredExe += ($_.Name)
257 }
258
259 # Remove exclusions
260 ForEach($e in $DiscoveredExe) {
261 $e = $e -replace '.exe$', ''
262 $bare_name = $e -replace '_je$', ''
263
264 if($ExcludeExesSet.Contains($bare_name)) {
265 Write-Warning "Test $e is excluded"
266 continue
267 }
268 MakeAndAdd -token $e -HashTable $TestExes
269 }
270
271 if($TestExes.Count -lt 1) {
272 Write-Error "Failed to discover test executables"
273 exit 1
274 }
275 }
276
277 # Ordered by exe @{ Exe = @{ TestCase = LogName }}
278 $CasesToRun = [ordered]@{}
279
280 if($SuiteRun -ne "") {
281 $suite_list = $SuiteRun -split ' '
282 ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
283 } elseif ($RunAll) {
284 # Discover all the test binaries
285 if($EnableJE) {
286 $pattern = "*_test_je.exe"
287 } else {
288 $pattern = "*_test.exe"
289 }
290
291 $search_path = -join ($BinariesFolder, $pattern)
292 Write-Host "Binaries Search Path: $search_path"
293
294 $ListOfExe = @()
295 dir -Path $search_path | ForEach-Object {
296 $ListOfExe += ($_.Name)
297 }
298
299 # Exclude those in RunOnly from running as suites
300 $ListOfSuites = @()
301 ForEach($e in $ListOfExe) {
302
303 $e = $e -replace '.exe$', ''
304 $bare_name = $e -replace '_je$', ''
305
306 if($ExcludeExesSet.Contains($bare_name)) {
307 Write-Warning "Test $e is excluded"
308 continue
309 }
310
311 if($RunOnly.Contains($bare_name)) {
312 MakeAndAdd -token $e -HashTable $TestExes
313 } else {
314 $ListOfSuites += $bare_name
315 }
316 }
317
318 ProcessSuites -ListOfSuites $ListOfSuites -HashOfHashes $CasesToRun
319 }
320
321
322 # Invoke a test with a filter and redirect all output
323 $InvokeTestCase = {
324 param($exe, $test, $log);
325 &$exe --gtest_filter=$test > $log 2>&1
326 }
327
328 # Invoke all tests and redirect output
329 $InvokeTestAsync = {
330 param($exe, $log)
331 &$exe > $log 2>&1
332 }
333
334 # Hash that contains tests to rerun if any failed
335 # Those tests will be rerun sequentially
336 # $Rerun = [ordered]@{}
337 # Test limiting factor here
338 [int]$count = 0
339 # Overall status
340 [bool]$script:success = $true;
341
RunJobs($Suites, $TestCmds, [int] $ConcurrencyVal)342 function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
343 {
344 # Array to wait for any of the running jobs
345 $jobs = @()
346 # Hash JobToLog
347 $JobToLog = @{}
348
349 # Wait for all to finish and get the results
350 while(($JobToLog.Count -gt 0) -or
351 ($TestCmds.Count -gt 0) -or
352 ($Suites.Count -gt 0)) {
353
354 # Make sure we have maximum concurrent jobs running if anything
355 # and the $Limit either not set or allows to proceed
356 while(($JobToLog.Count -lt $ConcurrencyVal) -and
357 ((($TestCmds.Count -gt 0) -or ($Suites.Count -gt 0)) -and
358 (($Limit -lt 0) -or ($count -lt $Limit)))) {
359
360 # We always favore suites to run if available
361 [string]$exe_name = ""
362 [string]$log_path = ""
363 $Cases = @{}
364
365 if($Suites.Count -gt 0) {
366 # Will the first one
367 ForEach($e in $Suites.Keys) {
368 $exe_name = $e
369 $Cases = $Suites[$e]
370 break
371 }
372 [string]$test_case = ""
373 [string]$log_path = ""
374 ForEach($c in $Cases.Keys) {
375 $test_case = $c
376 $log_path = $Cases[$c]
377 break
378 }
379
380 Write-Host "Starting $exe_name::$test_case"
381 [string]$Exe = -Join ($BinariesFolder, $exe_name)
382 $job = Start-Job -Name "$exe_name::$test_case" -ArgumentList @($Exe,$test_case,$log_path) -ScriptBlock $InvokeTestCase
383 $JobToLog.Add($job, $log_path)
384
385 $Cases.Remove($test_case)
386 if($Cases.Count -lt 1) {
387 $Suites.Remove($exe_name)
388 }
389
390 } elseif ($TestCmds.Count -gt 0) {
391
392 ForEach($e in $TestCmds.Keys) {
393 $exe_name = $e
394 $log_path = $TestCmds[$e]
395 break
396 }
397
398 Write-Host "Starting $exe_name"
399 [string]$Exe = -Join ($BinariesFolder, $exe_name)
400 $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
401 $JobToLog.Add($job, $log_path)
402
403 $TestCmds.Remove($exe_name)
404
405 } else {
406 Write-Error "In the job loop but nothing to run"
407 exit 1
408 }
409
410 ++$count
411 } # End of Job starting loop
412
413 if($JobToLog.Count -lt 1) {
414 break
415 }
416
417 $jobs = @()
418 foreach($k in $JobToLog.Keys) { $jobs += $k }
419
420 $completed = Wait-Job -Job $jobs -Any
421 $log = $JobToLog[$completed]
422 $JobToLog.Remove($completed)
423
424 $message = -join @($completed.Name, " State: ", ($completed.State))
425
426 $log_content = @(Get-Content $log)
427
428 if($completed.State -ne "Completed") {
429 $script:success = $false
430 Write-Warning $message
431 $log_content | Write-Warning
432 } else {
433 # Scan the log. If we find PASSED and no occurrence of FAILED
434 # then it is a success
435 [bool]$pass_found = $false
436 ForEach($l in $log_content) {
437
438 if(($l -match "^\[\s+FAILED") -or
439 ($l -match "Assertion failed:")) {
440 $pass_found = $false
441 break
442 }
443
444 if(($l -match "^\[\s+PASSED") -or
445 ($l -match " : PASSED$") -or
446 ($l -match "^PASS$") -or # Special c_test case
447 ($l -match "Passed all tests!") ) {
448 $pass_found = $true
449 }
450 }
451
452 if(!$pass_found) {
453 $script:success = $false;
454 Write-Warning $message
455 $log_content | Write-Warning
456 } else {
457 Write-Host $message
458 }
459 }
460
461 # Remove cached job info from the system
462 # Should be no output
463 Receive-Job -Job $completed | Out-Null
464 }
465 }
466
467 RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
468
469 $EndDate = (Get-Date)
470
471 New-TimeSpan -Start $StartDate -End $EndDate |
472 ForEach-Object {
473 "Elapsed time: {0:g}" -f $_
474 }
475
476
477 if(!$script:success) {
478 # This does not succeed killing off jobs quick
479 # So we simply exit
480 # Remove-Job -Job $jobs -Force
481 # indicate failure using this exit code
482 exit 1
483 }
484
485 exit 0
486
487
488