Code to change assert behaviour when running under a debugger

In Linux, you can create a __assert_fail routine which will be triggered when an assert is encountered in code compiled with asserts enabled. Using some debugger detection, you can have the code behave differently in that situation. The code to accomplish detection at load-time looks like:

#include <assert.h>
#include <stdio.h>
#include <signal.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/ptrace.h>

static int ptrace_failed;

static int
detect_ptrace(void)
{
    int status, waitrc;

    pid_t child = fork();
    if (child == -1) return -1;
    if (child == 0) {
        if (ptrace(PT_ATTACH, getppid(), 0, 0))
            exit(1);
        do {
            waitrc = waitpid(getppid(), &status, 0);
        } while (waitrc == -1 && errno == EINTR);
        ptrace(PT_DETACH, getppid(), (caddr_t)1, SIGCONT);
        exit(0);
    }
    do {
        waitrc = waitpid(child, &status, 0);
    } while (waitrc == -1 && errno == EINTR);
    return WEXITSTATUS(status);
}

__attribute__((constructor))
static void
detect_debugger(void)
{
    ptrace_failed = detect_ptrace();
}

void __assert_fail(const char * assertion, const char * file, unsigned int line, const char * function) {
    fprintf(stderr, "Assert: %s failed at %s:%d in function %s\n", assertion, file, line, function);
    if (ptrace_failed)
        raise(SIGTRAP);
    else
        abort();
}

This code, because it uses an __attribute__((constructor)) will detect a program being started under a debugger.

You could move the detect_debugger() call into the __assert_fail function, and this provides full run-time detection of the debugger and behaviour alteration. On the assumption that not a lot of asserts() get triggered, you have an effective behaviour alteration of your code when run under a debugger. In this case the __assert_fail looks like:

void __assert_fail(const char * assertion, const char * file, unsigned int line, const char * function) {
    fprintf(stderr, "Assert: %s failed at %s:%d in function %s\n", assertion, file, line, function);
    if (detect_ptrace())
        raise(SIGTRAP);
    else
        abort();
}