1 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
2 # This script enables you running RocksDB tests by running
3 # All the tests concurrently and utilizing all the cores
4 Param(
5 [switch]$EnableJE = $false, # Look for and use test executable, append _je to listed exclusions
6 [switch]$RunAll = $false, # Will attempt discover all *_test[_je].exe binaries and run all
7 # of them as Google suites. I.e. It will run test cases concurrently
8 # except those mentioned as $Run, those will run as individual test cases
9 # And any execlued with $ExcludeExes or $ExcludeCases
10 # It will also not run any individual test cases
11 # excluded but $ExcludeCasese
12 [switch]$RunAllExe = $false, # Look for and use test exdcutables, append _je to exclusions automatically
13 # It will attempt to run them in parallel w/o breaking them up on individual
14 # test cases. Those listed with $ExcludeExes will be excluded
15 [string]$SuiteRun = "", # Split test suites in test cases and run in parallel, not compatible with $RunAll
16 [string]$Run = "", # Run specified executables in parallel but do not split to test cases
17 [string]$ExcludeCases = "", # Exclude test cases, expects a comma separated list, no spaces
18 # Takes effect when $RunAll or $SuiteRun is specified. Must have full
19 # Test cases name including a group and a parameter if any
20 [string]$ExcludeExes = "", # Exclude exes from consideration, expects a comma separated list,
21 # no spaces. Takes effect only when $RunAll is specified
22 [string]$WorkFolder = "", # Direct tests to use that folder. SSD or Ram drive are better options.
23 # Number of async tasks that would run concurrently. Recommend a number below 64.
24 # However, CPU utlization really depends on the storage media. Recommend ram based disk.
25 # a value of 1 will run everything serially
26 [int]$Concurrency = 8,
27 [int]$Limit = -1 # -1 means do not limit for test purposes
28 )
29
30 # Folders and commands must be fullpath to run assuming
31 # the current folder is at the root of the git enlistment
32 $StartDate = (Get-Date)
33 $StartDate
34
35
36 $DebugPreference = "Continue"
37
38 # These tests are not google test suites and we should guard
39 # Against running them as suites
40 $RunOnly = New-Object System.Collections.Generic.HashSet[string]
41 $RunOnly.Add("c_test") | Out-Null
42 $RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
43 $RunOnly.Add("merge_test") | Out-Null
44 $RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
45 $RunOnly.Add("backupable_db_test") | Out-Null # Disabled
46 $RunOnly.Add("timer_queue_test") | Out-Null # Not a gtest
47
48 if($RunAll -and $SuiteRun -ne "") {
49 Write-Error "$RunAll and $SuiteRun are not compatible"
50 exit 1
51 }
52
53 if($RunAllExe -and $Run -ne "") {
54 Write-Error "$RunAllExe and $Run are not compatible"
55 exit 1
56 }
57
58 # If running under Appveyor assume that root
59 [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
60 if($Appveyor -ne "") {
61 $RootFolder = $Appveyor
62 } else {
63 $RootFolder = $PSScriptRoot -replace '\\build_tools', ''
64 }
65
66 $LogFolder = -Join($RootFolder, "\db_logs\")
67 $BinariesFolder = -Join($RootFolder, "\build\Debug\")
68
69 if($WorkFolder -eq "") {
70
71 # If TEST_TMPDIR is set use it
72 [string]$var = $Env:TEST_TMPDIR
73 if($var -eq "") {
74 $WorkFolder = -Join($RootFolder, "\db_tests\")
75 $Env:TEST_TMPDIR = $WorkFolder
76 } else {
77 $WorkFolder = $var
78 }
79 } else {
80 # Override from a command line
81 $Env:TEST_TMPDIR = $WorkFolder
82 }
83
84 Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
85 Write-Output "BinariesFolder: $BinariesFolder, LogFolder: $LogFolder"
86
87 # Create test directories in the current folder
88 md -Path $WorkFolder -ErrorAction Ignore | Out-Null
89 md -Path $LogFolder -ErrorAction Ignore | Out-Null
90
91
92 $ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
93 if($ExcludeCases -ne "") {
94 Write-Host "ExcludeCases: $ExcludeCases"
95 $l = $ExcludeCases -split ' '
96 ForEach($t in $l) {
97 $ExcludeCasesSet.Add($t) | Out-Null
98 }
99 }
100
101 $ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
102 if($ExcludeExes -ne "") {
103 Write-Host "ExcludeExe: $ExcludeExes"
104 $l = $ExcludeExes -split ' '
105 ForEach($t in $l) {
106 $ExcludeExesSet.Add($t) | Out-Null
107 }
108 }
109
110
111 # Extract the names of its tests by running db_test with --gtest_list_tests.
112 # This filter removes the "#"-introduced comments, and expands to
113 # fully-qualified names by changing input like this:
114 #
115 # DBTest.
116 # Empty
117 # WriteEmptyBatch
118 # MultiThreaded/MultiThreadedDBTest.
119 # MultiThreaded/0 # GetParam() = 0
120 # MultiThreaded/1 # GetParam() = 1
121 # RibbonTypeParamTest/0. # TypeParam = struct DefaultTypesAndSettings
122 # CompactnessAndBacktrackAndFpRate
123 # Extremes
124 # FindOccupancyForSuccessRate
125 #
126 # into this:
127 #
128 # DBTest.Empty
129 # DBTest.WriteEmptyBatch
130 # MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
131 # MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
132 # RibbonTypeParamTest/0.CompactnessAndBacktrackAndFpRate
133 # RibbonTypeParamTest/0.Extremes
134 # RibbonTypeParamTest/0.FindOccupancyForSuccessRate
135 #
136 # Output into the parameter in a form TestName -> Log File Name
ExtractTestCases([string]$GTestExe, $HashTable)137 function ExtractTestCases([string]$GTestExe, $HashTable) {
138
139 $Tests = @()
140 # Run db_test to get a list of tests and store it into $a array
141 &$GTestExe --gtest_list_tests | tee -Variable Tests | Out-Null
142
143 # Current group
144 $Group=""
145
146 ForEach( $l in $Tests) {
147
148 # remove trailing comment if any
149 $l = $l -replace '\s+\#.*',''
150 # Leading whitespace is fine
151 $l = $l -replace '^\s+',''
152 # Trailing dot is a test group but no whitespace
153 if ($l -match "\.$" -and $l -notmatch "\s+") {
154 $Group = $l
155 } else {
156 # Otherwise it is a test name, remove leading space
157 $test = $l
158 # create a log name
159 $test = "$Group$test"
160
161 if($ExcludeCasesSet.Contains($test)) {
162 Write-Warning "$test case is excluded"
163 continue
164 }
165
166 $test_log = $test -replace '[\./]','_'
167 $test_log += ".log"
168 $log_path = -join ($LogFolder, $test_log)
169
170 # Add to a hashtable
171 $HashTable.Add($test, $log_path);
172 }
173 }
174 }
175
176 # The function removes trailing .exe siffix if any,
177 # creates a name for the log file
178 # Then adds the test name if it was not excluded into
179 # a HashTable in a form of test_name -> log_path
MakeAndAdd([string] $token, $HashTable)180 function MakeAndAdd([string]$token, $HashTable) {
181
182 $test_name = $token -replace '.exe$', ''
183 $log_name = -join ($test_name, ".log")
184 $log_path = -join ($LogFolder, $log_name)
185 $HashTable.Add($test_name, $log_path)
186 }
187
188 # This function takes a list of Suites to run
189 # Lists all the test cases in each of the suite
190 # and populates HashOfHashes
191 # Ordered by suite(exe) @{ Exe = @{ TestCase = LogName }}
ProcessSuites($ListOfSuites, $HashOfHashes)192 function ProcessSuites($ListOfSuites, $HashOfHashes) {
193
194 $suite_list = $ListOfSuites
195 # Problem: if you run --gtest_list_tests on
196 # a non Google Test executable then it will start executing
197 # and we will get nowhere
198 ForEach($suite in $suite_list) {
199
200 if($RunOnly.Contains($suite)) {
201 Write-Warning "$suite is excluded from running as Google test suite"
202 continue
203 }
204
205 if($EnableJE) {
206 $suite += "_je"
207 }
208
209 $Cases = [ordered]@{}
210 $Cases.Clear()
211 $suite_exe = -Join ($BinariesFolder, $suite)
212 ExtractTestCases -GTestExe $suite_exe -HashTable $Cases
213 if($Cases.Count -gt 0) {
214 $HashOfHashes.Add($suite, $Cases);
215 }
216 }
217
218 # Make logs and run
219 if($CasesToRun.Count -lt 1) {
220 Write-Error "Failed to extract tests from $SuiteRun"
221 exit 1
222 }
223
224 }
225
226 # This will contain all test executables to run
227
228 # Hash table that contains all non suite
229 # Test executable to run
230 $TestExes = [ordered]@{}
231
232 # Check for test exe that are not
233 # Google Test Suites
234 # Since this is explicitely mentioned it is not subject
235 # for exclusions
236 if($Run -ne "") {
237
238 $test_list = $Run -split ' '
239 ForEach($t in $test_list) {
240
241 if($EnableJE) {
242 $t += "_je"
243 }
244 MakeAndAdd -token $t -HashTable $TestExes
245 }
246
247 if($TestExes.Count -lt 1) {
248 Write-Error "Failed to extract tests from $Run"
249 exit 1
250 }
251 } elseif($RunAllExe) {
252 # Discover all the test binaries
253 if($EnableJE) {
254 $pattern = "*_test_je.exe"
255 } else {
256 $pattern = "*_test.exe"
257 }
258
259 $search_path = -join ($BinariesFolder, $pattern)
260 Write-Host "Binaries Search Path: $search_path"
261
262 $DiscoveredExe = @()
263 dir -Path $search_path | ForEach-Object {
264 $DiscoveredExe += ($_.Name)
265 }
266
267 # Remove exclusions
268 ForEach($e in $DiscoveredExe) {
269 $e = $e -replace '.exe$', ''
270 $bare_name = $e -replace '_je$', ''
271
272 if($ExcludeExesSet.Contains($bare_name)) {
273 Write-Warning "Test $e is excluded"
274 continue
275 }
276 MakeAndAdd -token $e -HashTable $TestExes
277 }
278
279 if($TestExes.Count -lt 1) {
280 Write-Error "Failed to discover test executables"
281 exit 1
282 }
283 }
284
285 # Ordered by exe @{ Exe = @{ TestCase = LogName }}
286 $CasesToRun = [ordered]@{}
287
288 if($SuiteRun -ne "") {
289 $suite_list = $SuiteRun -split ' '
290 ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
291 } elseif ($RunAll) {
292 # Discover all the test binaries
293 if($EnableJE) {
294 $pattern = "*_test_je.exe"
295 } else {
296 $pattern = "*_test.exe"
297 }
298
299 $search_path = -join ($BinariesFolder, $pattern)
300 Write-Host "Binaries Search Path: $search_path"
301
302 $ListOfExe = @()
303 dir -Path $search_path | ForEach-Object {
304 $ListOfExe += ($_.Name)
305 }
306
307 # Exclude those in RunOnly from running as suites
308 $ListOfSuites = @()
309 ForEach($e in $ListOfExe) {
310
311 $e = $e -replace '.exe$', ''
312 $bare_name = $e -replace '_je$', ''
313
314 if($ExcludeExesSet.Contains($bare_name)) {
315 Write-Warning "Test $e is excluded"
316 continue
317 }
318
319 if($RunOnly.Contains($bare_name)) {
320 MakeAndAdd -token $e -HashTable $TestExes
321 } else {
322 $ListOfSuites += $bare_name
323 }
324 }
325
326 ProcessSuites -ListOfSuites $ListOfSuites -HashOfHashes $CasesToRun
327 }
328
329
330 # Invoke a test with a filter and redirect all output
331 $InvokeTestCase = {
332 param($exe, $test, $log);
333 &$exe --gtest_filter=$test > $log 2>&1
334 }
335
336 # Invoke all tests and redirect output
337 $InvokeTestAsync = {
338 param($exe, $log)
339 &$exe > $log 2>&1
340 }
341
342 # Hash that contains tests to rerun if any failed
343 # Those tests will be rerun sequentially
344 # $Rerun = [ordered]@{}
345 # Test limiting factor here
346 [int]$count = 0
347 # Overall status
348 [bool]$script:success = $true;
349
RunJobs($Suites, $TestCmds, [int] $ConcurrencyVal)350 function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
351 {
352 # Array to wait for any of the running jobs
353 $jobs = @()
354 # Hash JobToLog
355 $JobToLog = @{}
356
357 # Wait for all to finish and get the results
358 while(($JobToLog.Count -gt 0) -or
359 ($TestCmds.Count -gt 0) -or
360 ($Suites.Count -gt 0)) {
361
362 # Make sure we have maximum concurrent jobs running if anything
363 # and the $Limit either not set or allows to proceed
364 while(($JobToLog.Count -lt $ConcurrencyVal) -and
365 ((($TestCmds.Count -gt 0) -or ($Suites.Count -gt 0)) -and
366 (($Limit -lt 0) -or ($count -lt $Limit)))) {
367
368 # We always favore suites to run if available
369 [string]$exe_name = ""
370 [string]$log_path = ""
371 $Cases = @{}
372
373 if($Suites.Count -gt 0) {
374 # Will the first one
375 ForEach($e in $Suites.Keys) {
376 $exe_name = $e
377 $Cases = $Suites[$e]
378 break
379 }
380 [string]$test_case = ""
381 [string]$log_path = ""
382 ForEach($c in $Cases.Keys) {
383 $test_case = $c
384 $log_path = $Cases[$c]
385 break
386 }
387
388 Write-Host "Starting $exe_name::$test_case"
389 [string]$Exe = -Join ($BinariesFolder, $exe_name)
390 $job = Start-Job -Name "$exe_name::$test_case" -ArgumentList @($Exe,$test_case,$log_path) -ScriptBlock $InvokeTestCase
391 $JobToLog.Add($job, $log_path)
392
393 $Cases.Remove($test_case)
394 if($Cases.Count -lt 1) {
395 $Suites.Remove($exe_name)
396 }
397
398 } elseif ($TestCmds.Count -gt 0) {
399
400 ForEach($e in $TestCmds.Keys) {
401 $exe_name = $e
402 $log_path = $TestCmds[$e]
403 break
404 }
405
406 Write-Host "Starting $exe_name"
407 [string]$Exe = -Join ($BinariesFolder, $exe_name)
408 $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
409 $JobToLog.Add($job, $log_path)
410
411 $TestCmds.Remove($exe_name)
412
413 } else {
414 Write-Error "In the job loop but nothing to run"
415 exit 1
416 }
417
418 ++$count
419 } # End of Job starting loop
420
421 if($JobToLog.Count -lt 1) {
422 break
423 }
424
425 $jobs = @()
426 foreach($k in $JobToLog.Keys) { $jobs += $k }
427
428 $completed = Wait-Job -Job $jobs -Any
429 $log = $JobToLog[$completed]
430 $JobToLog.Remove($completed)
431
432 $message = -join @($completed.Name, " State: ", ($completed.State))
433
434 $log_content = @(Get-Content $log)
435
436 if($completed.State -ne "Completed") {
437 $script:success = $false
438 Write-Warning $message
439 $log_content | Write-Warning
440 } else {
441 # Scan the log. If we find PASSED and no occurrence of FAILED
442 # then it is a success
443 [bool]$pass_found = $false
444 ForEach($l in $log_content) {
445
446 if(($l -match "^\[\s+FAILED") -or
447 ($l -match "Assertion failed:")) {
448 $pass_found = $false
449 break
450 }
451
452 if(($l -match "^\[\s+PASSED") -or
453 ($l -match " : PASSED$") -or
454 ($l -match "^PASS$") -or # Special c_test case
455 ($l -match "Passed all tests!") ) {
456 $pass_found = $true
457 }
458 }
459
460 if(!$pass_found) {
461 $script:success = $false;
462 Write-Warning $message
463 $log_content | Write-Warning
464 } else {
465 Write-Host $message
466 }
467 }
468
469 # Remove cached job info from the system
470 # Should be no output
471 Receive-Job -Job $completed | Out-Null
472 }
473 }
474
475 RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
476
477 $EndDate = (Get-Date)
478
479 New-TimeSpan -Start $StartDate -End $EndDate |
480 ForEach-Object {
481 "Elapsed time: {0:g}" -f $_
482 }
483
484
485 if(!$script:success) {
486 # This does not succeed killing off jobs quick
487 # So we simply exit
488 # Remove-Job -Job $jobs -Force
489 # indicate failure using this exit code
490 exit 1
491 }
492
493 exit 0
494
495
496