1#!/usr/bin/env groovy 2@Library('StanUtils') 3import org.stan.Utils 4 5def runTests(String testPath, boolean jumbo = false) { 6 try { 7 if (jumbo) { 8 sh "./runTests.py -j${env.PARALLEL} ${testPath} --jumbo" 9 } else { 10 sh "./runTests.py -j${env.PARALLEL} ${testPath}" 11 } 12 } 13 finally { junit 'test/**/*.xml' } 14} 15 16def runTestsWin(String testPath, boolean buildLibs = true, boolean jumbo = false) { 17 withEnv(['PATH+TBB=./lib/tbb']) { 18 bat "echo $PATH" 19 if (buildLibs){ 20 bat "mingw32-make.exe -f make/standalone math-libs" 21 } 22 try { 23 if (jumbo) { 24 bat "runTests.py -j${env.PARALLEL} ${testPath} --jumbo" 25 } else { 26 bat "runTests.py -j${env.PARALLEL} ${testPath}" 27 } 28 } 29 finally { junit 'test/**/*.xml' } 30 } 31} 32 33 34def deleteDirWin() { 35 bat "attrib -r -s /s /d" 36 deleteDir() 37} 38 39def skipRemainingStages = false 40def skipOpenCL = false 41 42def utils = new org.stan.Utils() 43 44def isBranch(String b) { env.BRANCH_NAME == b } 45 46String alsoNotify() { 47 if (isBranch('master') || isBranch('develop')) { 48 "stan-buildbot@googlegroups.com" 49 } else "" 50} 51Boolean isPR() { env.CHANGE_URL != null } 52String fork() { env.CHANGE_FORK ?: "stan-dev" } 53String branchName() { isPR() ? env.CHANGE_BRANCH :env.BRANCH_NAME } 54String cmdstan_pr() { params.cmdstan_pr ?: ( env.CHANGE_TARGET == "master" ? "downstream_hotfix" : "downstream_tests" ) } 55String stan_pr() { params.stan_pr ?: ( env.CHANGE_TARGET == "master" ? "downstream_hotfix" : "downstream_tests" ) } 56 57pipeline { 58 agent none 59 parameters { 60 string(defaultValue: '', name: 'cmdstan_pr', description: 'PR to test CmdStan upstream against e.g. PR-630') 61 string(defaultValue: '', name: 'stan_pr', description: 'PR to test Stan upstream against e.g. PR-630') 62 booleanParam(defaultValue: false, name: 'withRowVector', description: 'Run additional distribution tests on RowVectors (takes 5x as long)') 63 booleanParam(defaultValue: false, name: 'run_win_tests', description: 'Run full unit tests on Windows.') 64 } 65 options { 66 skipDefaultCheckout() 67 preserveStashes(buildCount: 7) 68 } 69 environment { 70 STAN_NUM_THREADS = '4' 71 } 72 stages { 73 stage('Kill previous builds') { 74 when { 75 not { branch 'develop' } 76 not { branch 'master' } 77 } 78 steps { 79 script { 80 utils.killOldBuilds() 81 } 82 } 83 } 84 stage("Clang-format") { 85 agent any 86 steps { 87 sh "printenv" 88 deleteDir() 89 retry(3) { checkout scm } 90 withCredentials([usernamePassword(credentialsId: 'a630aebc-6861-4e69-b497-fd7f496ec46b', 91 usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD')]) { 92 sh """#!/bin/bash 93 set -x 94 git checkout -b ${branchName()} 95 clang-format --version 96 find stan test -name '*.hpp' -o -name '*.cpp' | xargs -n20 -P${env.PARALLEL} clang-format -i 97 if [[ `git diff` != "" ]]; then 98 git config --global user.email "mc.stanislaw@gmail.com" 99 git config --global user.name "Stan Jenkins" 100 git add stan test 101 git commit -m "[Jenkins] auto-formatting by `clang-format --version`" 102 git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${fork()}/math.git ${branchName()} 103 echo "Exiting build because clang-format found changes." 104 echo "Those changes are now found on stan-dev/math under branch ${branchName()}" 105 echo "Please 'git pull' before continuing to develop." 106 exit 1 107 fi""" 108 } 109 } 110 post { 111 always { deleteDir() } 112 failure { 113 script { 114 emailext ( 115 subject: "[StanJenkins] Autoformattted: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", 116 body: "Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' " + 117 "has been autoformatted and the changes committed " + 118 "to your branch, if permissions allowed." + 119 "Please pull these changes before continuing." + 120 "\n\n" + 121 "See https://github.com/stan-dev/stan/wiki/Coding-Style-and-Idioms" + 122 " for setting up the autoformatter locally.\n"+ 123 "(Check console output at ${env.BUILD_URL})", 124 recipientProviders: [[$class: 'RequesterRecipientProvider']], 125 to: "${env.CHANGE_AUTHOR_EMAIL}" 126 ) 127 } 128 } 129 } 130 } 131 stage('Linting & Doc checks') { 132 agent any 133 steps { 134 script { 135 deleteDir() 136 retry(3) { checkout scm } 137 sh "git clean -xffd" 138 stash 'MathSetup' 139 sh "echo CXX=${env.CXX} -Werror > make/local" 140 sh "echo BOOST_PARALLEL_JOBS=${env.PARALLEL} >> make/local" 141 parallel( 142 CppLint: { sh "make cpplint" }, 143 Dependencies: { sh """#!/bin/bash 144 set -o pipefail 145 make test-math-dependencies 2>&1 | tee dependencies.log""" } , 146 Documentation: { sh "make doxygen" }, 147 ) 148 } 149 } 150 post { 151 always { 152 recordIssues enabledForFailure: true, tools: 153 [cppLint(), 154 groovyScript(parserId: 'mathDependencies', pattern: '**/dependencies.log')] 155 deleteDir() 156 } 157 } 158 } 159 stage('Verify changes') { 160 agent { label 'linux' } 161 steps { 162 script { 163 164 retry(3) { checkout scm } 165 sh 'git clean -xffd' 166 167 def paths = ['stan', 'make', 'lib', 'test', 'runTests.py', 'runChecks.py', 'makefile', 'Jenkinsfile', '.clang-format'].join(" ") 168 skipRemainingStages = utils.verifyChanges(paths) 169 170 def openCLPaths = ['stan/math/opencl', 'test/unit/math/opencl'].join(" ") 171 skipOpenCL = utils.verifyChanges(openCLPaths) 172 } 173 } 174 } 175 stage('Headers check') { 176 when { 177 expression { 178 !skipRemainingStages 179 } 180 } 181 agent any 182 steps { 183 deleteDir() 184 unstash 'MathSetup' 185 sh "echo CXX=${env.CXX} -Werror > make/local" 186 sh "make -j${env.PARALLEL} test-headers" 187 } 188 post { always { deleteDir() } } 189 } 190 stage('Full Unit Tests') { 191 agent any 192 when { 193 expression { 194 !skipRemainingStages 195 } 196 } 197 steps { 198 deleteDir() 199 unstash 'MathSetup' 200 sh "echo CXXFLAGS += -fsanitize=address > make/local" 201 script { 202 if (isUnix()) { 203 runTests("test/unit", true) 204 } else { 205 runTestsWin("test/unit", true) 206 } 207 } 208 } 209 post { always { retry(3) { deleteDir() } } } 210 } 211 stage('Always-run tests') { 212 when { 213 expression { 214 !skipRemainingStages 215 } 216 } 217 failFast true 218 parallel { 219 stage('MPI tests') { 220 agent { label 'linux && mpi' } 221 steps { 222 deleteDir() 223 unstash 'MathSetup' 224 sh "echo CXX=${MPICXX} >> make/local" 225 sh "echo CXX_TYPE=gcc >> make/local" 226 sh "echo STAN_MPI=true >> make/local" 227 runTests("test/unit/math/prim/functor") 228 runTests("test/unit/math/rev/functor") 229 } 230 post { always { retry(3) { deleteDir() } } } 231 } 232 stage('OpenCL CPU tests') { 233 when { 234 expression { 235 !skipOpenCL 236 } 237 } 238 agent { label "linux-gpu" } 239 steps { 240 script { 241 if (isUnix()) { 242 deleteDir() 243 unstash 'MathSetup' 244 sh "echo CXX=${env.CXX} -Werror > make/local" 245 sh "echo STAN_OPENCL=true>> make/local" 246 sh "echo OPENCL_PLATFORM_ID=${env.OPENCL_PLATFORM_ID_CPU}>> make/local" 247 sh "echo OPENCL_DEVICE_ID=${env.OPENCL_DEVICE_ID_CPU}>> make/local" 248 // skips tests that require specific support in OpenCL 249 sh 'echo "ifdef NO_CPU_OPENCL_INT64_BASE_ATOMIC" >> make/local' 250 sh 'echo "CXXFLAGS += -DSTAN_TEST_SKIP_REQUIRING_OPENCL_INT64_BASE_ATOMIC" >> make/local' 251 sh 'echo "endif" >> make/local' 252 253 runTests("test/unit/math/opencl", false) 254 runTests("test/unit/multiple_translation_units_test.cpp") 255 } else { 256 deleteDirWin() 257 unstash 'MathSetup' 258 bat "echo CXX=${env.CXX} -Werror > make/local" 259 bat "echo STAN_OPENCL=true >> make/local" 260 bat "echo OPENCL_PLATFORM_ID=${env.OPENCL_PLATFORM_ID_CPU} >> make/local" 261 bat "echo OPENCL_DEVICE_ID=${env.OPENCL_DEVICE_ID_CPU} >> make/local" 262 bat 'echo LDFLAGS_OPENCL= -L"C:\\Program Files (x86)\\IntelSWTools\\system_studio_2020\\OpenCL\\sdk\\lib\\x64" -lOpenCL >> make/local' 263 bat "mingw32-make.exe -f make/standalone math-libs" 264 runTestsWin("test/unit/math/opencl", false, false) 265 runTestsWin("test/unit/multiple_translation_units_test.cpp", false, false) 266 } 267 } 268 } 269 } 270 stage('OpenCL GPU tests') { 271 agent { label "linux-gpu" } 272 steps { 273 script { 274 if (isUnix()) { 275 deleteDir() 276 unstash 'MathSetup' 277 sh "echo CXX=${env.CXX} -Werror > make/local" 278 sh "echo STAN_OPENCL=true>> make/local" 279 sh "echo OPENCL_PLATFORM_ID=${env.OPENCL_PLATFORM_ID_GPU} >> make/local" 280 sh "echo OPENCL_DEVICE_ID=${env.OPENCL_DEVICE_ID_GPU} >> make/local" 281 runTests("test/unit/math/opencl", false) 282 runTests("test/unit/multiple_translation_units_test.cpp") 283 } else { 284 deleteDirWin() 285 unstash 'MathSetup' 286 bat "echo CXX=${env.CXX} -Werror > make/local" 287 bat "echo STAN_OPENCL=true >> make/local" 288 bat "echo OPENCL_PLATFORM_ID=${env.OPENCL_PLATFORM_ID_GPU} >> make/local" 289 bat "echo OPENCL_DEVICE_ID=${env.OPENCL_DEVICE_ID_GPU} >> make/local" 290 bat 'echo LDFLAGS_OPENCL= -L"C:\\Program Files (x86)\\IntelSWTools\\system_studio_2020\\OpenCL\\sdk\\lib\\x64" -lOpenCL >> make/local' 291 bat "mingw32-make.exe -f make/standalone math-libs" 292 runTestsWin("test/unit/math/opencl", false, false) 293 runTestsWin("test/unit/multiple_translation_units_test.cpp", false, false) 294 } 295 296 } 297 } 298 } 299 stage('Distribution tests') { 300 agent { label "distribution-tests" } 301 steps { 302 deleteDir() 303 unstash 'MathSetup' 304 sh """ 305 echo CXX=${env.CXX} > make/local 306 echo O=0 >> make/local 307 echo N_TESTS=${env.N_TESTS} >> make/local 308 """ 309 script { 310 if (params.withRowVector || isBranch('develop') || isBranch('master')) { 311 sh "echo CXXFLAGS+=-DSTAN_TEST_ROW_VECTORS >> make/local" 312 sh "echo CXXFLAGS+=-DSTAN_PROB_TEST_ALL >> make/local" 313 } 314 } 315 sh "./runTests.py -j${env.PARALLEL} test/prob > dist.log 2>&1" 316 } 317 post { 318 always { 319 script { zip zipFile: "dist.log.zip", archive: true, glob: 'dist.log' } 320 retry(3) { deleteDir() } 321 } 322 failure { 323 echo "Distribution tests failed. Check out dist.log.zip artifact for test logs." 324 } 325 } 326 } 327 stage('Expressions test') { 328 agent any 329 steps { 330 unstash 'MathSetup' 331 script { 332 sh "echo O=0 > make/local" 333 sh "python ./test/code_generator_test.py" 334 sh "python ./test/signature_parser_test.py" 335 sh "python ./test/statement_types_test.py" 336 sh "python ./test/varmat_compatibility_summary_test.py" 337 sh "python ./test/varmat_compatibility_test.py" 338 withEnv(['PATH+TBB=./lib/tbb']) { 339 sh "python ./test/expressions/test_expression_testing_framework.py" 340 } 341 withEnv(['PATH+TBB=./lib/tbb']) { 342 try { sh "./runTests.py -j${env.PARALLEL} test/expressions" } 343 finally { junit 'test/**/*.xml' } 344 } 345 sh "make clean-all" 346 sh "echo STAN_THREADS=true >> make/local" 347 withEnv(['PATH+TBB=./lib/tbb']) { 348 try { 349 sh "./runTests.py -j${env.PARALLEL} test/expressions --only-functions reduce_sum map_rect" 350 } 351 finally { junit 'test/**/*.xml' } 352 } 353 } 354 } 355 post { always { deleteDir() } } 356 } 357 stage('Threading tests') { 358 agent any 359 steps { 360 script { 361 deleteDir() 362 unstash 'MathSetup' 363 sh "echo CXX=${env.CXX} -Werror > make/local" 364 sh "echo STAN_THREADS=true >> make/local" 365 sh "export STAN_NUM_THREADS=4" 366 if (isBranch('develop') || isBranch('master')) { 367 runTests("test/unit") 368 sh "find . -name *_test.xml | xargs rm" 369 } else { 370 runTests("test/unit -f thread") 371 sh "find . -name *_test.xml | xargs rm" 372 runTests("test/unit -f map_rect") 373 sh "find . -name *_test.xml | xargs rm" 374 runTests("test/unit -f reduce_sum") 375 } 376 } 377 } 378 post { always { retry(3) { deleteDir() } } } 379 } 380 stage('Windows Headers & Unit') { 381 when { 382 allOf { 383 anyOf { 384 branch 'develop' 385 branch 'master' 386 expression { params.run_win_tests } 387 } 388 expression { 389 !skipRemainingStages 390 } 391 } 392 } 393 agent { label 'windows' } 394 steps { 395 deleteDirWin() 396 unstash 'MathSetup' 397 bat "mingw32-make.exe -f make/standalone math-libs" 398 runTestsWin("test/unit", false, false) 399 } 400 } 401 } 402 } 403 stage('Upstream tests') { 404 when { 405 allOf { 406 expression { 407 env.BRANCH_NAME ==~ /PR-\d+/ 408 } 409 expression { 410 !skipRemainingStages 411 } 412 } 413 } 414 steps { 415 build(job: "Stan/${stan_pr()}", 416 parameters: [string(name: 'math_pr', value: env.BRANCH_NAME), 417 string(name: 'cmdstan_pr', value: cmdstan_pr())]) 418 } 419 } 420 stage('Upload doxygen') { 421 agent any 422 when { branch 'develop'} 423 steps { 424 deleteDir() 425 retry(3) { checkout scm } 426 withCredentials([usernamePassword(credentialsId: 'a630aebc-6861-4e69-b497-fd7f496ec46b', 427 usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD')]) { 428 sh """#!/bin/bash 429 set -x 430 make doxygen 431 git config --global user.email "mc.stanislaw@gmail.com" 432 git config --global user.name "Stan Jenkins" 433 git checkout --detach 434 git branch -D gh-pages 435 git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/stan-dev/math.git :gh-pages 436 git checkout --orphan gh-pages 437 git add -f doc 438 git commit -m "auto generated docs from Jenkins" 439 git subtree push --prefix doc/api/html https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/stan-dev/math.git gh-pages 440 """ 441 } 442 } 443 post { always { deleteDir() } } 444 } 445 } 446 post { 447 always { 448 node("osx || linux") { 449 recordIssues enabledForFailure: false, tool: clang() 450 } 451 } 452 success { 453 script { 454 utils.updateUpstream(env, 'stan') 455 utils.mailBuildResults("SUCCESSFUL") 456 } 457 } 458 unstable { script { utils.mailBuildResults("UNSTABLE", alsoNotify()) } } 459 failure { script { utils.mailBuildResults("FAILURE", alsoNotify()) } } 460 } 461} 462