Process-shared condition variable : how to recover after one process dies?

I'm working on a simple FIFO queue to synchronize multiple instances of a server process.

This is very similar to Linux synchronization with FIFO waiting queue, except dealing with multiple processes instead of threads. I adapted caf's ticket lock to use process-shared mutex and condition variable from a shared memory segment. It also handles timeouts in case one process dies while processing a request:

#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <pthread.h>
#include <signal.h>
#include <errno.h>

static inline void fail(char *str)
{
    perror(str);
    exit(1);
}

/***************************************************************************************************/

/* Simple ticket lock queue with pthreads
 * https://stackoverflow.com/questions/3050083/linux-synchronization-with-fifo-waiting-queue
 */

typedef struct ticket_lock {
    pthread_mutex_t mutex;
    pthread_cond_t  cond;
    int queue_head, queue_tail;
} ticket_lock_t;

static void
ticket_init(ticket_lock_t *t)
{
    pthread_mutexattr_t mattr;
    pthread_mutexattr_init(&mattr);
    pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
    pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST);
    pthread_mutex_init(&t->mutex, &mattr);
    pthread_mutexattr_destroy(&mattr);

    pthread_condattr_t cattr;
    pthread_condattr_init(&cattr);
    pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED);
    pthread_cond_init(&t->cond, &cattr);
    pthread_condattr_destroy(&cattr);

    t->queue_head = t->queue_tail = 0;
}

static void
ticket_broadcast(ticket_lock_t *ticket)
{
    pthread_cond_broadcast(&ticket->cond);
}

static int
ticket_lock(ticket_lock_t *ticket)
{
    pthread_mutex_lock(&ticket->mutex);
    int queue_me = ticket->queue_tail++;

    while (queue_me > ticket->queue_head) {
        time_t sec = time(NULL) + 5;  /* 5s timeout */
        struct timespec ts = { .tv_sec = sec, .tv_nsec = 0 };

        fprintf(stderr, "%i: waiting, current: %i  me: %i\n", getpid(), ticket->queue_head, queue_me);

        if (pthread_cond_timedwait(&ticket->cond, &ticket->mutex, &ts) == 0)
            continue;

        if (errno != ETIMEDOUT) fail("pthread_cond_timedwait");

        /* Timeout, kick current user... */
        fprintf(stderr, "kicking stale ticket %i\n", ticket->queue_head);
        ticket->queue_head++;
        ticket_broadcast(ticket);
    }

    pthread_mutex_unlock(&ticket->mutex);
    return queue_me;
}

static void
ticket_unlock(ticket_lock_t *ticket, int me)
{
    pthread_mutex_lock(&ticket->mutex);
    if (ticket->queue_head == me) {  /* Normal case: we haven't timed out. */
        ticket->queue_head++;
        ticket_broadcast(ticket);
    }
    pthread_mutex_unlock(&ticket->mutex);
}


/***************************************************************************************************/
/* Shared memory */

#define SHM_NAME    "fifo_sched"
#define SHM_MAGIC   0xdeadbeef

struct sched_shm {
    int  size;
    int  magic;
    int  ready;

    /* sched stuff */
    ticket_lock_t queue;
};

static unsigned int shm_size = 256;
static struct sched_shm *shm = 0;

/* Create new shared memory segment */
static void
create_shm()
{
       int fd = shm_open(SHM_NAME, O_RDWR | O_CREAT | O_TRUNC, 0644);
       assert(fd != -1);
       int r = ftruncate(fd, shm_size);  assert(r == 0);
       void *pt = mmap(0, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
       assert(pt != MAP_FAILED);
       fprintf(stderr, "Created shared memory.\n");

       shm = pt;
       memset(shm, 0, sizeof(*shm));
       shm->size = shm_size;
       shm->magic = SHM_MAGIC;
       shm->ready = 0;

       ticket_init(&shm->queue);

       shm->ready = 1;
}

/* Attach existing shared memory segment */
static int
attach_shm()
{
    int fd = shm_open(SHM_NAME, O_RDWR, 0);
    if (fd == -1)  return 0;  /* Doesn't exist yet... */

    shm = mmap(0, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (shm == MAP_FAILED)  fail("mmap");
    fprintf(stderr, "Mapped shared memory.\n");

    assert(shm->magic == SHM_MAGIC);
    assert(shm->ready);
    return 1;
}

static void
shm_init()
{  
    fprintf(stderr, "shm_init()\n");
    assert(shm_size >= sizeof(struct sched_shm));
    if (!attach_shm())
        create_shm();
}


/***************************************************************************************************/

int main()
{
    shm_init();

    while (1) {
        int ticket = ticket_lock(&shm->queue);
        printf("%i: start %i\n", getpid(), ticket);
        printf("%i: done  %i\n", getpid(), ticket);
        ticket_unlock(&shm->queue, ticket);
    }
    return 0;
}

This works well standalone and while adding extra processes:

$ gcc -g -Wall -std=gnu99 -o foo foo.c -lpthread -lrt
$ ./foo
$ ./foo     # (in other term)
...
26370: waiting, current: 134803  me: 134804
26370: start 134804
26370: done  134804
26370: waiting, current: 134805  me: 134806
26370: start 134806
26370: done  134806
26370: waiting, current: 134807  me: 134808

However killing the 2nd instance breaks pthread_cond_timedwait() in the 1st:

pthread_cond_timedwait: No such file or directory

Which makes sense in a way, the condition variable was tracking this process and it's not there anymore.

Surely there must be a way to recover from this ?

[too long for a comment]

 pthread_cond_timedwait: No such file or directory

Hu! :-)

The pthread_*() family of functions does not set errno to any error code but returns it.

So to get any usable results change this

    if (pthread_cond_timedwait(&ticket->cond, &ticket->mutex, &ts) == 0)
        continue;

    if (errno != ETIMEDOUT) fail("pthread_cond_timedwait");

to be

    if ((errno = pthread_cond_timedwait(&ticket->cond, &ticket->mutex, &ts)) == 0)
        continue;

    if (errno != ETIMEDOUT) fail("pthread_cond_timedwait");