1#!/usr/bin/env groovy
2@Library('StanUtils')
3import org.stan.Utils
4
5def runTests(String testPath, boolean jumbo = false) {
6    try {
7        if (jumbo) {
8            sh "./runTests.py -j${env.PARALLEL} ${testPath} --jumbo"
9        } else {
10            sh "./runTests.py -j${env.PARALLEL} ${testPath}"
11        }
12    }
13    finally { junit 'test/**/*.xml' }
14}
15
16def runTestsWin(String testPath, boolean buildLibs = true, boolean jumbo = false) {
17    withEnv(['PATH+TBB=./lib/tbb']) {
18       bat "echo $PATH"
19       if (buildLibs){
20           bat "mingw32-make.exe -f make/standalone math-libs"
21       }
22       try {
23           if (jumbo) {
24               bat "runTests.py -j${env.PARALLEL} ${testPath} --jumbo"
25            } else {
26               bat "runTests.py -j${env.PARALLEL} ${testPath}"
27            }
28       }
29       finally { junit 'test/**/*.xml' }
30    }
31}
32
33
34def deleteDirWin() {
35    bat "attrib -r -s /s /d"
36    deleteDir()
37}
38
39def skipRemainingStages = false
40def skipOpenCL = false
41
42def utils = new org.stan.Utils()
43
44def isBranch(String b) { env.BRANCH_NAME == b }
45
46String alsoNotify() {
47    if (isBranch('master') || isBranch('develop')) {
48        "stan-buildbot@googlegroups.com"
49    } else ""
50}
51Boolean isPR() { env.CHANGE_URL != null }
52String fork() { env.CHANGE_FORK ?: "stan-dev" }
53String branchName() { isPR() ? env.CHANGE_BRANCH :env.BRANCH_NAME }
54String cmdstan_pr() { params.cmdstan_pr ?: ( env.CHANGE_TARGET == "master" ? "downstream_hotfix" : "downstream_tests" ) }
55String stan_pr() { params.stan_pr ?: ( env.CHANGE_TARGET == "master" ? "downstream_hotfix" : "downstream_tests" ) }
56
57pipeline {
58    agent none
59    parameters {
60        string(defaultValue: '', name: 'cmdstan_pr', description: 'PR to test CmdStan upstream against e.g. PR-630')
61        string(defaultValue: '', name: 'stan_pr', description: 'PR to test Stan upstream against e.g. PR-630')
62        booleanParam(defaultValue: false, name: 'withRowVector', description: 'Run additional distribution tests on RowVectors (takes 5x as long)')
63        booleanParam(defaultValue: false, name: 'run_win_tests', description: 'Run full unit tests on Windows.')
64    }
65    options {
66        skipDefaultCheckout()
67        preserveStashes(buildCount: 7)
68    }
69    environment {
70        STAN_NUM_THREADS = '4'
71    }
72    stages {
73        stage('Kill previous builds') {
74            when {
75                not { branch 'develop' }
76                not { branch 'master' }
77            }
78            steps {
79                script {
80                    utils.killOldBuilds()
81                }
82            }
83        }
84        stage("Clang-format") {
85            agent any
86            steps {
87                sh "printenv"
88                deleteDir()
89                retry(3) { checkout scm }
90                withCredentials([usernamePassword(credentialsId: 'a630aebc-6861-4e69-b497-fd7f496ec46b',
91                    usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD')]) {
92                    sh """#!/bin/bash
93                        set -x
94                        git checkout -b ${branchName()}
95                        clang-format --version
96                        find stan test -name '*.hpp' -o -name '*.cpp' | xargs -n20 -P${env.PARALLEL} clang-format -i
97                        if [[ `git diff` != "" ]]; then
98                            git config --global user.email "mc.stanislaw@gmail.com"
99                            git config --global user.name "Stan Jenkins"
100                            git add stan test
101                            git commit -m "[Jenkins] auto-formatting by `clang-format --version`"
102                            git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${fork()}/math.git ${branchName()}
103                            echo "Exiting build because clang-format found changes."
104                            echo "Those changes are now found on stan-dev/math under branch ${branchName()}"
105                            echo "Please 'git pull' before continuing to develop."
106                            exit 1
107                        fi"""
108                }
109            }
110            post {
111                always { deleteDir() }
112                failure {
113                    script {
114                        emailext (
115                            subject: "[StanJenkins] Autoformattted: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
116                            body: "Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' " +
117                                "has been autoformatted and the changes committed " +
118                                "to your branch, if permissions allowed." +
119                                "Please pull these changes before continuing." +
120                                "\n\n" +
121                                "See https://github.com/stan-dev/stan/wiki/Coding-Style-and-Idioms" +
122                                " for setting up the autoformatter locally.\n"+
123                            "(Check console output at ${env.BUILD_URL})",
124                            recipientProviders: [[$class: 'RequesterRecipientProvider']],
125                            to: "${env.CHANGE_AUTHOR_EMAIL}"
126                        )
127                    }
128                }
129            }
130        }
131        stage('Linting & Doc checks') {
132            agent any
133            steps {
134                script {
135                    deleteDir()
136                    retry(3) { checkout scm }
137                    sh "git clean -xffd"
138                    stash 'MathSetup'
139                    sh "echo CXX=${env.CXX} -Werror > make/local"
140                    sh "echo BOOST_PARALLEL_JOBS=${env.PARALLEL} >> make/local"
141                    parallel(
142                        CppLint: { sh "make cpplint" },
143                        Dependencies: { sh """#!/bin/bash
144                            set -o pipefail
145                            make test-math-dependencies 2>&1 | tee dependencies.log""" } ,
146                        Documentation: { sh "make doxygen" },
147                    )
148                }
149            }
150            post {
151                always {
152                    recordIssues enabledForFailure: true, tools:
153                        [cppLint(),
154                         groovyScript(parserId: 'mathDependencies', pattern: '**/dependencies.log')]
155                    deleteDir()
156                }
157            }
158        }
159        stage('Verify changes') {
160            agent { label 'linux' }
161            steps {
162                script {
163
164                    retry(3) { checkout scm }
165                    sh 'git clean -xffd'
166
167                    def paths = ['stan', 'make', 'lib', 'test', 'runTests.py', 'runChecks.py', 'makefile', 'Jenkinsfile', '.clang-format'].join(" ")
168                    skipRemainingStages = utils.verifyChanges(paths)
169
170                    def openCLPaths = ['stan/math/opencl', 'test/unit/math/opencl'].join(" ")
171                    skipOpenCL = utils.verifyChanges(openCLPaths)
172                }
173            }
174        }
175        stage('Headers check') {
176            when {
177                expression {
178                    !skipRemainingStages
179                }
180            }
181            agent any
182            steps {
183                deleteDir()
184                unstash 'MathSetup'
185                sh "echo CXX=${env.CXX} -Werror > make/local"
186                sh "make -j${env.PARALLEL} test-headers"
187            }
188            post { always { deleteDir() } }
189        }
190        stage('Full Unit Tests') {
191            agent any
192            when {
193                expression {
194                    !skipRemainingStages
195                }
196            }
197            steps {
198                deleteDir()
199                unstash 'MathSetup'
200	            sh "echo CXXFLAGS += -fsanitize=address > make/local"
201                script {
202                    if (isUnix()) {
203                        runTests("test/unit", true)
204                    } else {
205                        runTestsWin("test/unit", true)
206                    }
207                }
208            }
209            post { always { retry(3) { deleteDir() } } }
210        }
211        stage('Always-run tests') {
212            when {
213                expression {
214                    !skipRemainingStages
215                }
216            }
217            failFast true
218            parallel {
219                stage('MPI tests') {
220                    agent { label 'linux && mpi' }
221                    steps {
222                        deleteDir()
223                        unstash 'MathSetup'
224                        sh "echo CXX=${MPICXX} >> make/local"
225                        sh "echo CXX_TYPE=gcc >> make/local"
226                        sh "echo STAN_MPI=true >> make/local"
227                        runTests("test/unit/math/prim/functor")
228                        runTests("test/unit/math/rev/functor")
229                    }
230                    post { always { retry(3) { deleteDir() } } }
231                }
232                stage('OpenCL CPU tests') {
233                    when {
234                        expression {
235                            !skipOpenCL
236                        }
237                    }
238                    agent { label "linux-gpu" }
239                    steps {
240                        script {
241                            if (isUnix()) {
242                                deleteDir()
243                                unstash 'MathSetup'
244                                sh "echo CXX=${env.CXX} -Werror > make/local"
245                                sh "echo STAN_OPENCL=true>> make/local"
246                                sh "echo OPENCL_PLATFORM_ID=${env.OPENCL_PLATFORM_ID_CPU}>> make/local"
247                                sh "echo OPENCL_DEVICE_ID=${env.OPENCL_DEVICE_ID_CPU}>> make/local"
248                                // skips tests that require specific support in OpenCL
249                                sh 'echo "ifdef NO_CPU_OPENCL_INT64_BASE_ATOMIC" >> make/local'
250                                sh 'echo "CXXFLAGS += -DSTAN_TEST_SKIP_REQUIRING_OPENCL_INT64_BASE_ATOMIC" >> make/local'
251                                sh 'echo "endif" >> make/local'
252
253                                runTests("test/unit/math/opencl", false)
254                                runTests("test/unit/multiple_translation_units_test.cpp")
255                            } else {
256                                deleteDirWin()
257                                unstash 'MathSetup'
258                                bat "echo CXX=${env.CXX} -Werror > make/local"
259                                bat "echo STAN_OPENCL=true >> make/local"
260                                bat "echo OPENCL_PLATFORM_ID=${env.OPENCL_PLATFORM_ID_CPU} >> make/local"
261                                bat "echo OPENCL_DEVICE_ID=${env.OPENCL_DEVICE_ID_CPU} >> make/local"
262                                bat 'echo LDFLAGS_OPENCL= -L"C:\\Program Files (x86)\\IntelSWTools\\system_studio_2020\\OpenCL\\sdk\\lib\\x64" -lOpenCL >> make/local'
263                                bat "mingw32-make.exe -f make/standalone math-libs"
264                                runTestsWin("test/unit/math/opencl", false, false)
265                                runTestsWin("test/unit/multiple_translation_units_test.cpp", false, false)
266                            }
267                        }
268                    }
269                }
270                stage('OpenCL GPU tests') {
271                    agent { label "linux-gpu" }
272                    steps {
273                        script {
274                            if (isUnix()) {
275                                deleteDir()
276                                unstash 'MathSetup'
277                                sh "echo CXX=${env.CXX} -Werror > make/local"
278                                sh "echo STAN_OPENCL=true>> make/local"
279                                sh "echo OPENCL_PLATFORM_ID=${env.OPENCL_PLATFORM_ID_GPU} >> make/local"
280                                sh "echo OPENCL_DEVICE_ID=${env.OPENCL_DEVICE_ID_GPU} >> make/local"
281                                runTests("test/unit/math/opencl", false)
282                                runTests("test/unit/multiple_translation_units_test.cpp")
283                            } else {
284                                deleteDirWin()
285                                unstash 'MathSetup'
286                                bat "echo CXX=${env.CXX} -Werror > make/local"
287                                bat "echo STAN_OPENCL=true >> make/local"
288                                bat "echo OPENCL_PLATFORM_ID=${env.OPENCL_PLATFORM_ID_GPU} >> make/local"
289                                bat "echo OPENCL_DEVICE_ID=${env.OPENCL_DEVICE_ID_GPU} >> make/local"
290                                bat 'echo LDFLAGS_OPENCL= -L"C:\\Program Files (x86)\\IntelSWTools\\system_studio_2020\\OpenCL\\sdk\\lib\\x64" -lOpenCL >> make/local'
291                                bat "mingw32-make.exe -f make/standalone math-libs"
292                                runTestsWin("test/unit/math/opencl", false, false)
293                                runTestsWin("test/unit/multiple_translation_units_test.cpp", false, false)
294                            }
295
296                        }
297                    }
298                }
299                stage('Distribution tests') {
300                    agent { label "distribution-tests" }
301                    steps {
302                        deleteDir()
303                        unstash 'MathSetup'
304                        sh """
305                            echo CXX=${env.CXX} > make/local
306                            echo O=0 >> make/local
307                            echo N_TESTS=${env.N_TESTS} >> make/local
308                            """
309                        script {
310                            if (params.withRowVector || isBranch('develop') || isBranch('master')) {
311                                sh "echo CXXFLAGS+=-DSTAN_TEST_ROW_VECTORS >> make/local"
312                                sh "echo CXXFLAGS+=-DSTAN_PROB_TEST_ALL >> make/local"
313                            }
314                        }
315                        sh "./runTests.py -j${env.PARALLEL} test/prob > dist.log 2>&1"
316                    }
317                    post {
318                        always {
319                            script { zip zipFile: "dist.log.zip", archive: true, glob: 'dist.log' }
320                            retry(3) { deleteDir() }
321                        }
322                        failure {
323                            echo "Distribution tests failed. Check out dist.log.zip artifact for test logs."
324                            }
325                    }
326                }
327	            stage('Expressions test') {
328                    agent any
329                    steps {
330                        unstash 'MathSetup'
331                        script {
332                            sh "echo O=0 > make/local"
333                            sh "python ./test/code_generator_test.py"
334                            sh "python ./test/signature_parser_test.py"
335                            sh "python ./test/statement_types_test.py"
336                            sh "python ./test/varmat_compatibility_summary_test.py"
337                            sh "python ./test/varmat_compatibility_test.py"
338                            withEnv(['PATH+TBB=./lib/tbb']) {
339                                sh "python ./test/expressions/test_expression_testing_framework.py"
340                            }
341                            withEnv(['PATH+TBB=./lib/tbb']) {
342                                try { sh "./runTests.py -j${env.PARALLEL} test/expressions" }
343                                finally { junit 'test/**/*.xml' }
344                            }
345                            sh "make clean-all"
346                            sh "echo STAN_THREADS=true >> make/local"
347                            withEnv(['PATH+TBB=./lib/tbb']) {
348                                try {
349                                    sh "./runTests.py -j${env.PARALLEL} test/expressions --only-functions reduce_sum map_rect"
350				                }
351                                finally { junit 'test/**/*.xml' }
352                            }
353                        }
354                    }
355                    post { always { deleteDir() } }
356                }
357                stage('Threading tests') {
358                    agent any
359                    steps {
360                        script {
361                            deleteDir()
362                            unstash 'MathSetup'
363                            sh "echo CXX=${env.CXX} -Werror > make/local"
364                            sh "echo STAN_THREADS=true >> make/local"
365                            sh "export STAN_NUM_THREADS=4"
366                            if (isBranch('develop') || isBranch('master')) {
367                                runTests("test/unit")
368                                sh "find . -name *_test.xml | xargs rm"
369                            } else {
370                                runTests("test/unit -f thread")
371                                sh "find . -name *_test.xml | xargs rm"
372                                runTests("test/unit -f map_rect")
373                                sh "find . -name *_test.xml | xargs rm"
374                                runTests("test/unit -f reduce_sum")
375                            }
376                        }
377                    }
378                    post { always { retry(3) { deleteDir() } } }
379                }
380                stage('Windows Headers & Unit') {
381                    when {
382                        allOf {
383                            anyOf {
384                                branch 'develop'
385                                branch 'master'
386                                expression { params.run_win_tests }
387                            }
388                            expression {
389                                !skipRemainingStages
390                            }
391                        }
392                    }
393                    agent { label 'windows' }
394                    steps {
395                        deleteDirWin()
396                        unstash 'MathSetup'
397                        bat "mingw32-make.exe -f make/standalone math-libs"
398                        runTestsWin("test/unit", false, false)
399                    }
400                }
401            }
402        }
403        stage('Upstream tests') {
404            when {
405                allOf {
406                    expression {
407                        env.BRANCH_NAME ==~ /PR-\d+/
408                    }
409                    expression {
410                        !skipRemainingStages
411                    }
412                }
413            }
414            steps {
415                build(job: "Stan/${stan_pr()}",
416                        parameters: [string(name: 'math_pr', value: env.BRANCH_NAME),
417                                    string(name: 'cmdstan_pr', value: cmdstan_pr())])
418            }
419        }
420        stage('Upload doxygen') {
421            agent any
422            when { branch 'develop'}
423            steps {
424                deleteDir()
425                retry(3) { checkout scm }
426                withCredentials([usernamePassword(credentialsId: 'a630aebc-6861-4e69-b497-fd7f496ec46b',
427                                                  usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD')]) {
428                    sh """#!/bin/bash
429                        set -x
430                        make doxygen
431                        git config --global user.email "mc.stanislaw@gmail.com"
432                        git config --global user.name "Stan Jenkins"
433                        git checkout --detach
434                        git branch -D gh-pages
435                        git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/stan-dev/math.git :gh-pages
436                        git checkout --orphan gh-pages
437                        git add -f doc
438                        git commit -m "auto generated docs from Jenkins"
439                        git subtree push --prefix doc/api/html https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/stan-dev/math.git gh-pages
440                        """
441                }
442            }
443            post { always { deleteDir() } }
444        }
445    }
446    post {
447        always {
448            node("osx || linux") {
449                recordIssues enabledForFailure: false, tool: clang()
450            }
451        }
452        success {
453            script {
454                utils.updateUpstream(env, 'stan')
455                utils.mailBuildResults("SUCCESSFUL")
456            }
457        }
458        unstable { script { utils.mailBuildResults("UNSTABLE", alsoNotify()) } }
459        failure { script { utils.mailBuildResults("FAILURE", alsoNotify()) } }
460    }
461}
462