1#!/bin/bash
2#
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this
5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7########################################################################
8#
9# mozilla/security/nss/tests/all.sh
10#
11# Script to start selected available NSS QA suites on one machine
12# this script is called or sourced by NSS QA which runs on all required
13# platforms
14#
15# Needs to work on all Unix and Windows platforms
16#
17# Currently available NSS QA suites:
18# ----------------------------------
19#   cipher.sh    - tests NSS ciphers
20#   libpkix.sh   - tests PKIX functionality
21#   cert.sh      - exercises certutil and creates certs necessary for
22#                  all other tests
23#   dbtests.sh   - tests related to certificate databases
24#   tools.sh     - tests the majority of the NSS tools
25#   fips.sh      - tests basic functionallity of NSS in FIPS-compliant
26#                - mode
27#   sdr.sh       - tests NSS SDR
28#   crmf.sh      - CRMF/CMMF testing
29#   smime.sh     - S/MIME testing
30#   ssl.sh       - tests SSL V2 SSL V3 and TLS
31#   ocsp.sh      - OCSP testing
32#   merge.sh     - tests merging old and new shareable databases
33#   pkits.sh     - NIST/PKITS tests
34#   chains.sh    - PKIX cert chains tests
35#   dbupgrade.sh - upgrade databases to new shareable version (used
36#                  only in upgrade test cycle)
37#   memleak.sh   - memory leak testing (optional)
38#   ssl_gtests.sh- Gtest based unit tests for ssl
39#   gtests.sh    - Gtest based unit tests for everything else
40#   policy.sh    - Crypto Policy tests
41#   bogo.sh      - Bogo interop tests (disabled by default)
42#                  https://boringssl.googlesource.com/boringssl/+/master/ssl/test/PORTING.md
43#   interop.sh   - Interoperability tests (disabled by default)
44#                  https://github.com/ekr/tls_interop
45#   tlsfuzzer.sh - tlsfuzzer interop tests (disabled by default)
46#                  https://github.com/tomato42/tlsfuzzer/
47#
48# NSS testing is now devided to 4 cycles:
49# ---------------------------------------
50#   standard     - run test suites with defaults settings
51#   pkix         - run test suites with PKIX enabled
52#   upgradedb    - upgrade existing certificate databases to shareable
53#                  format (creates them if doesn't exist yet) and run
54#                  test suites with those databases. Requires to enable libdm.
55#   sharedb      - run test suites with shareable database format
56#                  enabled (databases are created directly to this
57#                  format). This is the default and doesn't need to be run separately.
58#   threadunsafe - run test suites with thread unsafe environment variable
59#                  so simulate running NSS locking for PKCS #11 modules which
60#                  are not thread safe.
61#
62# Mandatory environment variables (to be set before testing):
63# -----------------------------------------------------------
64#   HOST         - test machine host name
65#   DOMSUF       - test machine domain name
66#
67# Optional environment variables to specify build to use:
68# -------------------------------------------------------
69#   BUILT_OPT    - use optimized/debug build
70#   USE_64       - use 64bit/32bit build
71#
72# Optional environment variables to select which cycles/suites to test:
73# ---------------------------------------------------------------------
74#   NSS_CYCLES     - list of cycles to run (separated by space
75#                    character)
76#                  - by default all cycles are tested
77#
78#   NSS_TESTS      - list of all test suites to run (separated by space
79#                    character, without trailing .sh)
80#                  - this list can be reduced for individual test cycles
81#   NSS_THREAD_TESTS - list of test suites run in the threadunsafe cycle
82#
83#   NSS_SSL_TESTS  - list of ssl tests to run (see ssl.sh)
84#   NSS_SSL_RUN    - list of ssl sub-tests to run (see ssl.sh)
85#
86# Testing schema:
87# ---------------
88#                           all.sh                       ~  (main)
89#                              |                               |
90#          +------------+------------+-----------+---    ~  run_cycles
91#          |            |            |           |             |
92#      standard       pkix       upgradedb     sharedb   ~  run_cycle_*
93#         ...           |           ...         ...            |
94#                +------+------+------+----->            ~  run_tests
95#                |      |      |      |                        |
96#              cert   tools   fips   ssl   ...           ~  . *.sh
97#
98# Special strings:
99# ----------------
100#   FIXME ... known problems, search for this string
101#   NOTE .... unexpected behavior
102#
103# NOTE:
104# -----
105#   Unlike the old QA this is based on files sourcing each other
106#   This is done to save time, since a great portion of time is lost
107#   in calling and sourcing the same things multiple times over the
108#   network. Also, this way all scripts have all shell function
109#   available and a completely common environment
110#
111########################################################################
112
113RUN_FIPS=""
114
115############################## run_tests ###############################
116# run test suites defined in TESTS variable, skip scripts defined in
117# TESTS_SKIP variable
118########################################################################
119run_tests()
120{
121    echo "Running test cycle: ${TEST_MODE} ----------------------"
122    echo "List of tests that will be executed: ${TESTS}"
123    for TEST in ${TESTS}
124    do
125        # NOTE: the spaces are important. If you don't include
126        # the spaces, then turning off ssl_gtests will also turn off ssl
127        # tests.
128        echo " ${TESTS_SKIP} " | grep " ${TEST} " > /dev/null
129        if [ $? -eq 0 ]; then
130            continue
131        fi
132
133        SCRIPTNAME=${TEST}.sh
134        echo "Running tests for ${TEST}"
135        echo "TIMESTAMP ${TEST} BEGIN: `date`"
136        (cd ${QADIR}/${TEST}; . ./${SCRIPTNAME} 2>&1)
137        echo "TIMESTAMP ${TEST} END: `date`"
138    done
139}
140
141########################## run_cycle_standard ##########################
142# run test suites with sql database (no PKIX)
143########################################################################
144run_cycle_standard()
145{
146    TEST_MODE=STANDARD
147
148    TESTS="${ALL_TESTS}"
149    TESTS_SKIP="libpkix pkits"
150
151    NSS_DEFAULT_DB_TYPE=${NSS_DEFAULT_DB_TYPE:-"sql"}
152    export NSS_DEFAULT_DB_TYPE
153
154    run_tests
155}
156
157############################ run_cycle_pkix ############################
158# run test suites with PKIX enabled
159########################################################################
160run_cycle_pkix()
161{
162    TEST_MODE=PKIX
163
164    TABLE_ARGS="bgcolor=cyan"
165    html_head "Testing with PKIX"
166    html "</TABLE><BR>"
167
168    HOSTDIR="${HOSTDIR}/pkix"
169    mkdir -p "${HOSTDIR}"
170    init_directories
171
172    NSS_ENABLE_PKIX_VERIFY="1"
173    export NSS_ENABLE_PKIX_VERIFY
174
175    TESTS="${ALL_TESTS}"
176    TESTS_SKIP="cipher dbtests sdr crmf smime merge multinit"
177
178    export -n NSS_SSL_RUN
179
180    # use the default format. (unset for the shell, export -n for binaries)
181    export -n NSS_DEFAULT_DB_TYPE
182    unset NSS_DEFAULT_DB_TYPE
183
184    run_tests
185}
186
187######################### run_cycle_upgrade_db #########################
188# upgrades certificate database to shareable format and run test suites
189# with those databases
190########################################################################
191run_cycle_upgrade_db()
192{
193    TEST_MODE=UPGRADE_DB
194
195    TABLE_ARGS="bgcolor=pink"
196    html_head "Testing with upgraded library"
197    html "</TABLE><BR>"
198
199    OLDHOSTDIR="${HOSTDIR}"
200    HOSTDIR="${HOSTDIR}/upgradedb"
201    mkdir -p "${HOSTDIR}"
202    init_directories
203
204    if [ -r "${OLDHOSTDIR}/cert.log" ]; then
205        DIRS="alicedir bobdir CA cert_extensions client clientCA dave eccurves eve ext_client ext_server $RUN_FIPS SDR server serverCA stapling tools/copydir cert.log cert.done tests.*"
206        for i in $DIRS
207        do
208            cp -r ${OLDHOSTDIR}/${i} ${HOSTDIR} #2> /dev/null
209        done
210    fi
211
212    # upgrade certs dbs to shared db
213    TESTS="dbupgrade"
214    TESTS_SKIP=
215
216    run_tests
217
218    NSS_DEFAULT_DB_TYPE="sql"
219    export NSS_DEFAULT_DB_TYPE
220
221    # run the subset of tests with the upgraded database
222    TESTS="${ALL_TESTS}"
223    TESTS_SKIP="cipher libpkix cert dbtests sdr ocsp pkits chains"
224
225    run_tests
226}
227
228########################## run_cycle_shared_db #########################
229# run test suites with certificate databases set to shareable format
230########################################################################
231run_cycle_shared_db()
232{
233    TEST_MODE=SHARED_DB
234
235    TABLE_ARGS="bgcolor=yellow"
236    html_head "Testing with shared library"
237    html "</TABLE><BR>"
238
239    HOSTDIR="${HOSTDIR}/sharedb"
240    mkdir -p "${HOSTDIR}"
241    init_directories
242
243    NSS_DEFAULT_DB_TYPE="sql"
244    export NSS_DEFAULT_DB_TYPE
245
246    # run the tests for native sharedb support
247    TESTS="${ALL_TESTS}"
248    TESTS_SKIP="dbupgrade"
249
250    export -n NSS_SSL_TESTS
251    export -n NSS_SSL_RUN
252
253    run_tests
254}
255
256########################## run_thread_unsafe #########################
257# run test suites with an non-thread safe softoken
258# This simulates loading a non-threadsafe PKCS #11 module and makes
259# Sure we don't have any deadlocks in our locking code
260########################################################################
261run_cycle_thread_unsafe()
262{
263    TEST_MODE=THREAD_UNSAFE
264
265    TABLE_ARGS="bgcolor=lightgray"
266    html_head "Testing with non-threadsafe softoken"
267    html "</TABLE><BR>"
268
269    HOSTDIR="${HOSTDIR}/threadunsafe"
270    mkdir -p "${HOSTDIR}"
271    init_directories
272
273    NSS_FORCE_TOKEN_LOCK=1
274    export NSS_FORCE_TOKEN_LOCK
275
276    # run the tests for appropriate for thread unsafe
277    # basically it's the ssl tests right now.
278    TESTS="${THREAD_TESTS}"
279    TESTS_SKIP="dbupgrade"
280
281    export -n NSS_SSL_TESTS
282    export -n NSS_SSL_RUN
283
284    run_tests
285}
286
287############################# run_cycles ###############################
288# run test cycles defined in CYCLES variable
289########################################################################
290run_cycles()
291{
292    for CYCLE in ${CYCLES}
293    do
294        case "${CYCLE}" in
295        "standard")
296            run_cycle_standard
297            ;;
298        "pkix")
299            if [ -z "$NSS_DISABLE_LIBPKIX" ]; then
300                run_cycle_pkix
301            fi
302            ;;
303        "upgradedb")
304            run_cycle_upgrade_db
305            ;;
306        "sharedb")
307            run_cycle_shared_db
308            ;;
309        "threadunsafe")
310            run_cycle_thread_unsafe
311            ;;
312        esac
313        . ${ENV_BACKUP}
314    done
315}
316
317############################## main code ###############################
318
319SCRIPTNAME=all.sh
320CLEANUP="${SCRIPTNAME}"
321cd `dirname $0`
322
323# all.sh should be the first one to try to source the init
324if [ -z "${INIT_SOURCED}" -o "${INIT_SOURCED}" != "TRUE" ]; then
325    cd common
326    . ./init.sh
327fi
328
329cycles="standard pkix threadunsafe"
330CYCLES=${NSS_CYCLES:-$cycles}
331
332NO_INIT_SUPPORT=`certutil --build-flags |grep -cw NSS_NO_INIT_SUPPORT`
333if [ $NO_INIT_SUPPORT -eq 0 ]; then
334    RUN_FIPS="fips"
335fi
336
337tests="cipher lowhash libpkix cert dbtests tools $RUN_FIPS sdr crmf smime ssl ocsp merge pkits ec gtests ssl_gtests policy"
338thread_tests="ssl ssl_gtests"
339# Don't run chains tests when we have a gyp build.
340if [ "$OBJDIR" != "Debug" -a "$OBJDIR" != "Release" ]; then
341  tests="$tests chains"
342fi
343TESTS=${NSS_TESTS:-$tests}
344
345ALL_TESTS=${TESTS}
346default_thread=""
347for i in ${ALL_TESTS}
348do
349    for j in ${thread_tests}
350    do
351        if [ $i = $j ]; then
352            default_thread="$default_thread $i"
353        fi
354    done
355done
356THREAD_TESTS=${NSS_THREAD_TESTS-$default_thread}
357
358nss_ssl_tests="crl iopr policy normal_normal"
359if [ $NO_INIT_SUPPORT -eq 0 ]; then
360    nss_ssl_tests="$nss_ssl_tests fips_normal normal_fips"
361fi
362NSS_SSL_TESTS="${NSS_SSL_TESTS:-$nss_ssl_tests}"
363
364# NOTE: 'stress' run is omitted by default
365nss_ssl_run="cov auth stapling signed_cert_timestamps scheme"
366NSS_SSL_RUN="${NSS_SSL_RUN:-$nss_ssl_run}"
367
368# NOTE:
369# Lists of enabled tests and other settings are stored to ${ENV_BACKUP}
370# file and are are restored after every test cycle.
371
372ENV_BACKUP=${HOSTDIR}/env.sh
373env_backup > ${ENV_BACKUP}
374
375# Print hardware support if we built it.
376if [ -f ${BINDIR}/hw-support ]; then
377    ${BINDIR}/hw-support
378fi
379
380if [ "${O_CRON}" = "ON" ]; then
381    run_cycles >> ${LOGFILE}
382else
383    run_cycles | tee -a ${LOGFILE}
384fi
385
386SCRIPTNAME=all.sh
387
388. ${QADIR}/common/cleanup.sh
389