tests/tap/basic.c

/*
 * Some utility routines for writing tests.
 *
 * Here are a variety of utility routines for writing tests compatible with
 * the TAP protocol.  All routines of the form ok() or is*() take a test
 * number and some number of appropriate arguments, check to be sure the
 * results match the expected output using the arguments, and print out
 * something appropriate for that test number.  Other utility routines help in
 * constructing more complex tests, skipping tests, reporting errors, setting
 * up the TAP output format, or finding things in the test environment.
 *
 * This file is part of C TAP Harness.  The current version plus supporting
 * documentation is at <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
 *
 * Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015
 *     Russ Allbery <eagle@eyrie.org>
 * Copyright 2001, 2002, 2004, 2005, 2006, 2007, 2008, 2011, 2012, 2013, 2014
 *     The Board of Trustees of the Leland Stanford Junior University
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include <errno.h>
#include <limits.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef _WIN32
# include <direct.h>
#else
# include <sys/stat.h>
#endif
#include <sys/types.h>
#include <unistd.h>

#include <tests/tap/basic.h>

/* Windows provides mkdir and rmdir under different names. */
#ifdef _WIN32
# define mkdir(p, m) _mkdir(p)
# define rmdir(p)    _rmdir(p)
#endif

/*
 * The test count.  Always contains the number that will be used for the next
 * test status.  This is exported to callers of the library.
 */
unsigned long testnum = 1;

/*
 * Status information stored so that we can give a test summary at the end of
 * the test case.  We store the planned final test and the count of failures.
 * We can get the highest test count from testnum.
 */
static unsigned long _planned = 0;
static unsigned long _failed  = 0;

/*
 * Store the PID of the process that called plan() and only summarize
 * results when that process exits, so as to not misreport results in forked
 * processes.
 */
static pid_t _process = 0;

/*
 * If true, we're doing lazy planning and will print out the plan based on the
 * last test number at the end of testing.
 */
static int _lazy = 0;

/*
 * If true, the test was aborted by calling bail().  Currently, this is only
 * used to ensure that we pass a false value to any cleanup functions even if
 * all tests to that point have passed.
 */
static int _aborted = 0;

/*
 * Registered cleanup functions.  These are stored as a linked list and run in
 * registered order by finish when the test program exits.  Each function is
 * passed a boolean value indicating whether all tests were successful.
 */
struct cleanup_func {
    test_cleanup_func func;
    struct cleanup_func *next;
};
static struct cleanup_func *cleanup_funcs = NULL;

/*
 * Registered diag files.  Any output found in these files will be printed out
 * as if it were passed to diag() before any other output we do.  This allows
 * background processes to log to a file and have that output interleaved with
 * the test output.
 */
struct diag_file {
    char *name;
    FILE *file;
    char *buffer;
    size_t bufsize;
    struct diag_file *next;
};
static struct diag_file *diag_files = NULL;

/*
 * Print a specified prefix and then the test description.  Handles turning
 * the argument list into a va_args structure suitable for passing to
 * print_desc, which has to be done in a macro.  Assumes that format is the
 * argument immediately before the variadic arguments.
 */
#define PRINT_DESC(prefix, format)              \
    do {                                        \
        if (format != NULL) {                   \
            va_list args;                       \
            if (prefix != NULL)                 \
                printf("%s", prefix);           \
            va_start(args, format);             \
            vprintf(format, args);              \
            va_end(args);                       \
        }                                       \
    } while (0)


/*
 * Form a new string by concatenating multiple strings.  The arguments must be
 * terminated by (const char *) 0.
 *
 * This function only exists because we can't assume asprintf.  We can't
 * simulate asprintf with snprintf because we're only assuming SUSv3, which
 * does not require that snprintf with a NULL buffer return the required
 * length.  When those constraints are relaxed, this should be ripped out and
 * replaced with asprintf or a more trivial replacement with snprintf.
 */
static char *
concat(const char *first, ...)
{
    va_list args;
    char *result;
    const char *string;
    size_t offset;
    size_t length = 0;

    /*
     * Find the total memory required.  Ensure we don't overflow length.  See
     * the comment for breallocarray for why we're using UINT_MAX here.
     */
    va_start(args, first);
    for (string = first; string != NULL; string = va_arg(args, const char *)) {
        if (length >= UINT_MAX - strlen(string))
            bail("strings too long in concat");
        length += strlen(string);
    }
    va_end(args);
    length++;

    /* Create the string. */
    result = bmalloc(length);
    va_start(args, first);
    offset = 0;
    for (string = first; string != NULL; string = va_arg(args, const char *)) {
        memcpy(result + offset, string, strlen(string));
        offset += strlen(string);
    }
    va_end(args);
    result[offset] = '\0';
    return result;
}


/*
 * Check all registered diag_files for any output.  We only print out the
 * output if we see a complete line; otherwise, we wait for the next newline.
 */
static void
check_diag_files(void)
{
    struct diag_file *file;
    fpos_t where;
    size_t length;
    int size, incomplete;

    /*
     * Walk through each file and read each line of output available.  The
     * general scheme here used is as follows: try to read a line of output at
     * a time.  If we get NULL, check for EOF; on EOF, advance to the next
     * file.
     *
     * If we get some data, see if it ends in a newline.  If it doesn't end in
     * a newline, we have one of two cases: our buffer isn't large enough, in
     * which case we resize it and try again, or we have incomplete data in
     * the file, in which case we rewind the file and will try again next
     * time.
     */
    for (file = diag_files; file != NULL; file = file->next) {
        clearerr(file->file);

        /* Store the current position in case we have to rewind. */
        if (fgetpos(file->file, &where) < 0)
            sysbail("cannot get position in %s", file->name);

        /* Continue until we get EOF or an incomplete line of data. */
        incomplete = 0;
        while (!feof(file->file) && !incomplete) {
            size = file->bufsize > INT_MAX ? INT_MAX : (int) file->bufsize;
            if (fgets(file->buffer, size, file->file) == NULL) {
                if (ferror(file->file))
                    sysbail("cannot read from %s", file->name);
                continue;
            }

            /*
             * See if the line ends in a newline.  If not, see which error
             * case we have.  Use UINT_MAX as a substitute for SIZE_MAX (see
             * the comment for breallocarray).
             */
            length = strlen(file->buffer);
            if (file->buffer[length - 1] != '\n') {
                if (length < file->bufsize - 1)
                    incomplete = 1;
                else {
                    if (file->bufsize >= UINT_MAX - BUFSIZ)
                        sysbail("line too long in %s", file->name);
                    file->bufsize += BUFSIZ;
                    file->buffer = brealloc(file->buffer, file->bufsize);
                }

                /*
                 * On either incomplete lines or too small of a buffer, rewind
                 * and read the file again (on the next pass, if incomplete).
                 * It's simpler than trying to double-buffer the file.
                 */
                if (fsetpos(file->file, &where) < 0)
                    sysbail("cannot set position in %s", file->name);
                continue;
            }

            /* We saw a complete line.  Print it out. */
            printf("# %s", file->buffer);
        }
    }
}


/*
 * Our exit handler.  Called on completion of the test to report a summary of
 * results provided we're still in the original process.  This also handles
 * printing out the plan if we used plan_lazy(), although that's suppressed if
 * we never ran a test (due to an early bail, for example), and running any
 * registered cleanup functions.
 */
static void
finish(void)
{
    int success, primary;
    struct cleanup_func *current;
    unsigned long highest = testnum - 1;
    struct diag_file *file, *tmp;

    /* Check for pending diag_file output. */
    check_diag_files();

    /* Free the diag_files. */
    file = diag_files;
    while (file != NULL) {
        tmp = file;
        file = file->next;
        fclose(tmp->file);
        free(tmp->name);
        free(tmp->buffer);
        free(tmp);
    }
    diag_files = NULL;

    /*
     * Determine whether all tests were successful, which is needed before
     * calling cleanup functions since we pass that fact to the functions.
     */
    if (_planned == 0 && _lazy)
        _planned = highest;
    success = (!_aborted && _planned == highest && _failed == 0);

    /*
     * If there are any registered cleanup functions, we run those first.  We
     * always run them, even if we didn't run a test.  Don't do anything
     * except free the diag_files and call cleanup functions if we aren't the
     * primary process (the process in which plan or plan_lazy was called),
     * and tell the cleanup functions that fact.
     */
    primary = (_process == 0 || getpid() == _process);
    while (cleanup_funcs != NULL) {
        cleanup_funcs->func(success, primary);
        current = cleanup_funcs;
        cleanup_funcs = cleanup_funcs->next;
        free(current);
    }
    if (!primary)
        return;

    /* Don't do anything further if we never planned a test. */
    if (_planned == 0)
        return;

    /* If we're aborting due to bail, don't print summaries. */
    if (_aborted)
        return;

    /* Print out the lazy plan if needed. */
    fflush(stderr);
    if (_lazy && _planned > 0)
        printf("1..%lu\n", _planned);

    /* Print out a summary of the results. */
    if (_planned > highest)
        diag("Looks like you planned %lu test%s but only ran %lu", _planned,
             (_planned > 1 ? "s" : ""), highest);
    else if (_planned < highest)
        diag("Looks like you planned %lu test%s but ran %lu extra", _planned,
             (_planned > 1 ? "s" : ""), highest - _planned);
    else if (_failed > 0)
        diag("Looks like you failed %lu test%s of %lu", _failed,
             (_failed > 1 ? "s" : ""), _planned);
    else if (_planned != 1)
        diag("All %lu tests successful or skipped", _planned);
    else
        diag("%lu test successful or skipped", _planned);
}


/*
 * Initialize things.  Turns on line buffering on stdout and then prints out
 * the number of tests in the test suite.  We intentionally don't check for
 * pending diag_file output here, since it should really come after the plan.
 */
void
plan(unsigned long count)
{
    if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
        sysdiag("cannot set stdout to line buffered");
    fflush(stderr);
    printf("1..%lu\n", count);
    testnum = 1;
    _planned = count;
    _process = getpid();
    if (atexit(finish) != 0) {
        sysdiag("cannot register exit handler");
        diag("cleanups will not be run");
    }
}


/*
 * Initialize things for lazy planning, where we'll automatically print out a
 * plan at the end of the program.  Turns on line buffering on stdout as well.
 */
void
plan_lazy(void)
{
    if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
        sysdiag("cannot set stdout to line buffered");
    testnum = 1;
    _process = getpid();
    _lazy = 1;
    if (atexit(finish) != 0)
        sysbail("cannot register exit handler to display plan");
}


/*
 * Skip the entire test suite and exits.  Should be called instead of plan(),
 * not after it, since it prints out a special plan line.  Ignore diag_file
 * output here, since it's not clear if it's allowed before the plan.
 */
void
skip_all(const char *format, ...)
{
    fflush(stderr);
    printf("1..0 # skip");
    PRINT_DESC(" ", format);
    putchar('\n');
    exit(0);
}


/*
 * Takes a boolean success value and assumes the test passes if that value
 * is true and fails if that value is false.
 */
int
ok(int success, const char *format, ...)
{
    fflush(stderr);
    check_diag_files();
    printf("%sok %lu", success ? "" : "not ", testnum++);
    if (!success)
        _failed++;
    PRINT_DESC(" - ", format);
    putchar('\n');
    return success;
}


/*
 * Same as ok(), but takes the format arguments as a va_list.
 */
int
okv(int success, const char *format, va_list args)
{
    fflush(stderr);
    check_diag_files();
    printf("%sok %lu", success ? "" : "not ", testnum++);
    if (!success)
        _failed++;
    if (format != NULL) {
        printf(" - ");
        vprintf(format, args);
    }
    putchar('\n');
    return success;
}


/*
 * Skip a test.
 */
void
skip(const char *reason, ...)
{
    fflush(stderr);
    check_diag_files();
    printf("ok %lu # skip", testnum++);
    PRINT_DESC(" ", reason);
    putchar('\n');
}


/*
 * Report the same status on the next count tests.
 */
int
ok_block(unsigned long count, int success, const char *format, ...)
{
    unsigned long i;

    fflush(stderr);
    check_diag_files();
    for (i = 0; i < count; i++) {
        printf("%sok %lu", success ? "" : "not ", testnum++);
        if (!success)
            _failed++;
        PRINT_DESC(" - ", format);
        putchar('\n');
    }
    return success;
}


/*
 * Skip the next count tests.
 */
void
skip_block(unsigned long count, const char *reason, ...)
{
    unsigned long i;

    fflush(stderr);
    check_diag_files();
    for (i = 0; i < count; i++) {
        printf("ok %lu # skip", testnum++);
        PRINT_DESC(" ", reason);
        putchar('\n');
    }
}


/*
 * Takes an expected integer and a seen integer and assumes the test passes
 * if those two numbers match.
 */
int
is_int(long wanted, long seen, const char *format, ...)
{
    int success;

    fflush(stderr);
    check_diag_files();
    success = (wanted == seen);
    if (success)
        printf("ok %lu", testnum++);
    else {
        diag("wanted: %ld", wanted);
        diag("  seen: %ld", seen);
        printf("not ok %lu", testnum++);
        _failed++;
    }
    PRINT_DESC(" - ", format);
    putchar('\n');
    return success;
}


/*
 * Takes a string and what the string should be, and assumes the test passes
 * if those strings match (using strcmp).
 */
int
is_string(const char *wanted, const char *seen, const char *format, ...)
{
    int success;

    if (wanted == NULL)
        wanted = "(null)";
    if (seen == NULL)
        seen = "(null)";
    fflush(stderr);
    check_diag_files();
    success = (strcmp(wanted, seen) == 0);
    if (success)
        printf("ok %lu", testnum++);
    else {
        diag("wanted: %s", wanted);
        diag("  seen: %s", seen);
        printf("not ok %lu", testnum++);
        _failed++;
    }
    PRINT_DESC(" - ", format);
    putchar('\n');
    return success;
}


/*
 * Takes an expected unsigned long and a seen unsigned long and assumes the
 * test passes if the two numbers match.  Otherwise, reports them in hex.
 */
int
is_hex(unsigned long wanted, unsigned long seen, const char *format, ...)
{
    int success;

    fflush(stderr);
    check_diag_files();
    success = (wanted == seen);
    if (success)
        printf("ok %lu", testnum++);
    else {
        diag("wanted: %lx", (unsigned long) wanted);
        diag("  seen: %lx", (unsigned long) seen);
        printf("not ok %lu", testnum++);
        _failed++;
    }
    PRINT_DESC(" - ", format);
    putchar('\n');
    return success;
}


/*
 * Bail out with an error.
 */
void
bail(const char *format, ...)
{
    va_list args;

    _aborted = 1;
    fflush(stderr);
    check_diag_files();
    fflush(stdout);
    printf("Bail out! ");
    va_start(args, format);
    vprintf(format, args);
    va_end(args);
    printf("\n");
    exit(255);
}


/*
 * Bail out with an error, appending strerror(errno).
 */
void
sysbail(const char *format, ...)
{
    va_list args;
    int oerrno = errno;

    _aborted = 1;
    fflush(stderr);
    check_diag_files();
    fflush(stdout);
    printf("Bail out! ");
    va_start(args, format);
    vprintf(format, args);
    va_end(args);
    printf(": %s\n", strerror(oerrno));
    exit(255);
}


/*
 * Report a diagnostic to stderr.  Always returns 1 to allow embedding in
 * compound statements.
 */
int
diag(const char *format, ...)
{
    va_list args;

    fflush(stderr);
    check_diag_files();
    fflush(stdout);
    printf("# ");
    va_start(args, format);
    vprintf(format, args);
    va_end(args);
    printf("\n");
    return 1;
}


/*
 * Report a diagnostic to stderr, appending strerror(errno).  Always returns 1
 * to allow embedding in compound statements.
 */
int
sysdiag(const char *format, ...)
{
    va_list args;
    int oerrno = errno;

    fflush(stderr);
    check_diag_files();
    fflush(stdout);
    printf("# ");
    va_start(args, format);
    vprintf(format, args);
    va_end(args);
    printf(": %s\n", strerror(oerrno));
    return 1;
}


/*
 * Register a new file for diag_file processing.
 */
void
diag_file_add(const char *name)
{
    struct diag_file *file, *prev;

    file = bcalloc(1, sizeof(struct diag_file));
    file->name = bstrdup(name);
    file->file = fopen(file->name, "r");
    if (file->file == NULL)
        sysbail("cannot open %s", name);
    file->buffer = bmalloc(BUFSIZ);
    file->bufsize = BUFSIZ;
    if (diag_files == NULL)
        diag_files = file;
    else {
        for (prev = diag_files; prev->next != NULL; prev = prev->next)
            ;
        prev->next = file;
    }
}


/*
 * Remove a file from diag_file processing.  If the file is not found, do
 * nothing, since there are some situations where it can be removed twice
 * (such as if it's removed from a cleanup function, since cleanup functions
 * are called after freeing all the diag_files).
 */
void
diag_file_remove(const char *name)
{
    struct diag_file *file;
    struct diag_file **prev = &diag_files;

    for (file = diag_files; file != NULL; file = file->next) {
        if (strcmp(file->name, name) == 0) {
            *prev = file->next;
            fclose(file->file);
            free(file->name);
            free(file->buffer);
            free(file);
            return;
        }
        prev = &file->next;
    }
}


/*
 * Allocate cleared memory, reporting a fatal error with bail on failure.
 */
void *
bcalloc(size_t n, size_t size)
{
    void *p;

    p = calloc(n, size);
    if (p == NULL)
        sysbail("failed to calloc %lu", (unsigned long)(n * size));
    return p;
}


/*
 * Allocate memory, reporting a fatal error with bail on failure.
 */
void *
bmalloc(size_t size)
{
    void *p;

    p = malloc(size);
    if (p == NULL)
        sysbail("failed to malloc %lu", (unsigned long) size);
    return p;
}


/*
 * Reallocate memory, reporting a fatal error with bail on failure.
 */
void *
brealloc(void *p, size_t size)
{
    p = realloc(p, size);
    if (p == NULL)
        sysbail("failed to realloc %lu bytes", (unsigned long) size);
    return p;
}


/*
 * The same as brealloc, but determine the size by multiplying an element
 * count by a size, similar to calloc.  The multiplication is checked for
 * integer overflow.
 *
 * We should technically use SIZE_MAX here for the overflow check, but
 * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not
 * guarantee that it exists.  They do guarantee that UINT_MAX exists, and we
 * can assume that UINT_MAX <= SIZE_MAX.
 *
 * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but
 * I disbelieve in the existence of such systems and they will have to cope
 * without overflow checks.)
 */
void *
breallocarray(void *p, size_t n, size_t size)
{
    if (n > 0 && UINT_MAX / n <= size)
        bail("reallocarray too large");
    p = realloc(p, n * size);
    if (p == NULL)
        sysbail("failed to realloc %lu bytes", (unsigned long) (n * size));
    return p;
}


/*
 * Copy a string, reporting a fatal error with bail on failure.
 */
char *
bstrdup(const char *s)
{
    char *p;
    size_t len;

    len = strlen(s) + 1;
    p = malloc(len);
    if (p == NULL)
        sysbail("failed to strdup %lu bytes", (unsigned long) len);
    memcpy(p, s, len);
    return p;
}


/*
 * Copy up to n characters of a string, reporting a fatal error with bail on
 * failure.  Don't use the system strndup function, since it may not exist and
 * the TAP library doesn't assume any portability support.
 */
char *
bstrndup(const char *s, size_t n)
{
    const char *p;
    char *copy;
    size_t length;

    /* Don't assume that the source string is nul-terminated. */
    for (p = s; (size_t) (p - s) < n && *p != '\0'; p++)
        ;
    length = (size_t) (p - s);
    copy = malloc(length + 1);
    if (p == NULL)
        sysbail("failed to strndup %lu bytes", (unsigned long) length);
    memcpy(copy, s, length);
    copy[length] = '\0';
    return copy;
}


/*
 * Locate a test file.  Given the partial path to a file, look under BUILD and
 * then SOURCE for the file and return the full path to the file.  Returns
 * NULL if the file doesn't exist.  A non-NULL return should be freed with
 * test_file_path_free().
 */
char *
test_file_path(const char *file)
{
    char *base;
    char *path = NULL;
    const char *envs[] = { "BUILD", "SOURCE", NULL };
    int i;

    for (i = 0; envs[i] != NULL; i++) {
        base = getenv(envs[i]);
        if (base == NULL)
            continue;
        path = concat(base, "/", file, (const char *) 0);
        if (access(path, R_OK) == 0)
            break;
        free(path);
        path = NULL;
    }
    return path;
}


/*
 * Free a path returned from test_file_path().  This function exists primarily
 * for Windows, where memory must be freed from the same library domain that
 * it was allocated from.
 */
void
test_file_path_free(char *path)
{
    free(path);
}


/*
 * Create a temporary directory, tmp, under BUILD if set and the current
 * directory if it does not.  Returns the path to the temporary directory in
 * newly allocated memory, and calls bail on any failure.  The return value
 * should be freed with test_tmpdir_free.
 *
 * This function uses sprintf because it attempts to be independent of all
 * other portability layers.  The use immediately after a memory allocation
 * should be safe without using snprintf or strlcpy/strlcat.
 */
char *
test_tmpdir(void)
{
    const char *build;
    char *path = NULL;

    build = getenv("BUILD");
    if (build == NULL)
        build = ".";
    path = concat(build, "/tmp", (const char *) 0);
    if (access(path, X_OK) < 0)
        if (mkdir(path, 0777) < 0)
            sysbail("error creating temporary directory %s", path);
    return path;
}


/*
 * Free a path returned from test_tmpdir() and attempt to remove the
 * directory.  If we can't delete the directory, don't worry; something else
 * that hasn't yet cleaned up may still be using it.
 */
void
test_tmpdir_free(char *path)
{
    if (path != NULL)
        rmdir(path);
    free(path);
}


/*
 * Register a cleanup function that is called when testing ends.  All such
 * registered functions will be run by finish.
 */
void
test_cleanup_register(test_cleanup_func func)
{
    struct cleanup_func *cleanup, **last;

    cleanup = bmalloc(sizeof(struct cleanup_func));
    cleanup->func = func;
    cleanup->next = NULL;
    last = &cleanup_funcs;
    while (*last != NULL)
        last = &(*last)->next;
    *last = cleanup;
}