/*
 * $Header: /san01/cvs/ashpool/csrc/continuity/mecha/Attic/thr.c,v 1.24 2004/05/27 13:55:00 aleigh Exp $ 
 */

/*
 * Copyright (c) 1994, 2004 Alex Leigh, All Rights Reserved
 * 
 * THIS IS UNPUBLISHED PROPRIETARY SOURCE CODE OF ALEX LEIGH The copyright
 * notice above does not evidence any actual or intended publication of such
 * source code.
 * 
 * Disclosure, copying, distribution, or transmission of of this code without
 * express permission is prohibited.
 */
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <stdint.h>
#include <pthread.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <errno.h>
#include <sys/time.h>

#include "mecha.h"

/* #define PRECREATE */

struct thrSinfo {
    struct thrStpool *tpool;
    int index;
};
typedef struct thrSinfo thrTinfo;

struct thrStrigger {
    unsigned int time;
    int fd;
    pthread_t tid;
};

typedef struct tpool_work {
    void (*routine) (void *data);
    void *arg;
    struct tpool_work *next;
} thrTtpool_work;

struct thrStpool {
    /* pool characteristics */
    int num_threads;
    int max_queue_size;
    int started;
    int inuse;

    int blocking_mode;
    /* pool state */
    int cur_queue_size;
    pthread_t *threads;
    thrTtpool_work *queue_head;
    thrTtpool_work *queue_tail;
    int queue_closed;
    int shutdown;
    /* pool synchronization */
    pthread_mutex_t queue_lock;
    pthread_mutex_t thread_lock;
    pthread_cond_t queue_not_empty;
    pthread_cond_t queue_not_full;
    pthread_cond_t queue_empty;
    struct thrStrigger *trigger;
};

pthread_key_t thrKtrigger;
void *thrFthread(void *);

void thrFmod_init(void)
{
    if (pthread_key_create(&thrKtrigger, NULL) == -1) {
        logFmsg(3, "thrFmod_init: Failed to create trigger TLS key.");
    }
}

int thrFsleep(int ms)
{
    struct timespec then;
    int ret;

#ifdef LINUX

    then.tv_sec = ms / 1000;
    then.tv_nsec = (ms % 1000) * 1000000;

    /* shut up vgrind, not required */
    ret = nanosleep (&then, NULL);

#else
    pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
    pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
    struct timeval now;
    int deltams;

    gettimeofday(&now, NULL);

    deltams = (now.tv_usec / 1000) + ms;

    then.tv_sec = now.tv_sec + (deltams / 1000);
    then.tv_nsec = (deltams % 1000) * 1000000;

    /*
     * OpenBSD requires that the mutex be locked, or else cond_timedwait will
     * return EINVAL. We can get away with this on Linux so far even though
     * it is bad behavior. Will generate valgrind warnings.
     */
#if defined(OPENBSD) || defined(FREEBSD)
    pthread_mutex_lock(&mut);
    pthread_mutex_lock(&mut);
#endif

    ret = pthread_cond_timedwait(&cond, &mut, (struct timespec *) &then);
#endif /* LINUX */

    return 0;
}

thrTtrigger *thrFgettrigger(void)
{
    return pthread_getspecific(thrKtrigger);
}

int thrFtrigger_set(unsigned int delta, int fd)
{
    thrTtrigger *trig = thrFgettrigger();

    if (trig == NULL) {
//        logFmsg(3, "trig NULL tid %d", pthread_self());
        return -1;
    }
    trig->fd = fd;
    trig->time = utlFtime() + delta;

    return 0;
}

int thrFtrigger_clear(void)
{
    thrTtrigger *trig = thrFgettrigger();

    if (trig == NULL)
        return -1;

    trig->time = 0;

    return 0;


}

void thrFcheck_triggers(thrTtpool * tp)
{
    int i;
    unsigned int now = utlFtime();

    for (i = 0; i < tp->num_threads; i++) {
        if (tp->trigger[i].time <= now && tp->trigger[i].time != 0) {
            /*
             * logFmsg(3, "thrFcheck_triggers: Triggering fd %d",
             * tp->trigger[i].fd);
             */
            shutdown(tp->trigger[i].fd, 2);
            tp->trigger[i].time = 0;
        }
    }
}

/*
 * This thread runs continuously and keeps statistics about the threadpool
 * current. It is also responsable for outputting messages when the pool
 * filles.
 */

void thrFstat_thr(void *tp)
{
    thrTtpool *tpool = (thrTtpool *) tp;

    while (1) {
        if (tpool->cur_queue_size > 0) {
            logFmsg(0, "thr_pool: [queue: %d/%d] [threads: %d/%d (%d)]",
                    tpool->cur_queue_size, tpool->max_queue_size,
                    tpool->inuse, tpool->started, tpool->num_threads);
        }
        thrFcheck_triggers(tpool);
        thrFsleep(1000);
    }
}

thrTtpool *thrFinit(int num_worker_threads,
                    int max_queue_size, int blocking_mode)
{
    unsigned int i;
    int ret;
    thrTtpool *tpool;

    /* allocate a pool data structure */
    if ((tpool = (thrTtpool *) malloc(sizeof(thrTtpool))) == NULL) {
        logFmsg(3, "thrFinit: malloc()");
        exit(1);
    }
    /* initialize th fields */
    tpool->num_threads = num_worker_threads;
    tpool->max_queue_size = max_queue_size;
    tpool->blocking_mode = blocking_mode;
    tpool->started = 1;
    tpool->inuse = 0;

    if ((tpool->threads =
         (pthread_t *) malloc(sizeof(pthread_t) * num_worker_threads))
        == NULL)
        perror("malloc"), exit(1);
    tpool->cur_queue_size = 0;
    tpool->queue_head = NULL;
    tpool->queue_tail = NULL;
    tpool->queue_closed = 0;
    tpool->shutdown = 0;

    tpool->trigger =
        (thrTtrigger *) malloc(sizeof(thrTtrigger) * num_worker_threads);
    memFclear(tpool->trigger, sizeof(thrTtrigger) * num_worker_threads);

    /* Start the statistics output thread */
    pthread_create((pthread_t *) & i, NULL,
                   (void *(*)(void *)) thrFstat_thr, (void *) tpool);

    if ((ret = pthread_mutex_init(&(tpool->queue_lock), NULL)) != 0) {
        logFmsg(2, "thrFinit: pthread_mutex_init failed. (e: %d)", ret);
        exit(1);
    }
    if ((ret = pthread_mutex_init(&(tpool->thread_lock), NULL)) != 0) {
        logFmsg(2, "thrFinit: pthread_mutex_init failed. (e: %d)", ret);
        exit(1);
    }
    if ((ret = pthread_cond_init(&(tpool->queue_not_empty), NULL)) != 0) {
        logFmsg(2, "thrFinit: conditional not_empty failed. (e: %d)", ret);
        exit(1);
    }
    if ((ret = pthread_cond_init(&(tpool->queue_not_full), NULL)) != 0) {
        logFmsg(2, "thrFinit: conditional not_full failed. (e: %d)", ret);
        exit(1);
    }
    if ((ret = pthread_cond_init(&(tpool->queue_empty), NULL)) != 0) {
        logFmsg(2, "thrFinit: conditional empty failed. (e: %d)", ret);
        exit(1);
    }
    {
        thrTinfo *info = (thrTinfo *) malloc(sizeof(thrTinfo));
        pthread_t tid;

        info->index = 0;
        info->tpool = tpool;

        if ((ret =
             pthread_create((pthread_t *) & tid, NULL, thrFthread,
                            (void *) info)) != 0) {
            logFmsg(2,
                    "thrFinit: Unable to create first pool thread. (e: %d)",
                    ret);
            exit(1);
        }
    }

#ifdef PRECREATE
    for (i = 1; i != num_worker_threads; i++) {
        thrTinfo *info = (thrTinfo *) malloc(sizeof(thrTinfo));
        pthread_t tid;

        info->index = i;
        info->tpool = tpool;

        if ((ret = pthread_create((pthread_t *) & tid,
                                  NULL, thrFthread, (void *) info)) != 0) {
            logFmsg(2, "thrFinit: Unable to create pool thread: %d", ret);
            exit(1);
        }
    }
#endif

    return tpool;
}

/*
 * Beyond, Amon Hen is before us.
 */

int thrFadd_work(thrTtpool * tpool, void (*routine) (void *data),
                 void *arg)
{
    int ret;
    thrTtpool_work *workp;

    if ((ret = pthread_mutex_lock(&(tpool->queue_lock))) != 0) {
        logFmsg(2, "thrFadd_work: mutex lock failed. (e: %d)", ret);
        exit(1);
    }
    /*
     * If the blocking mode is 1 and the queue is full than we are going to
     * return immediately. Otherwise we will go ahead and try to add the work
     * but will block until it is able to be accepted.
     */

    if ((tpool->cur_queue_size == tpool->max_queue_size) &&
        tpool->blocking_mode == 1) {
        if ((ret = pthread_mutex_unlock(&(tpool->queue_lock))) != 0) {
            logFmsg(2, "thrFadd_work: mutex unlock failed. (e: %d)", ret);
            exit(1);
        }
        return -1;
    }
    while ((tpool->cur_queue_size == tpool->max_queue_size) &&
           (!(tpool->shutdown || tpool->queue_closed))) {
        if ((ret = pthread_cond_wait(&(tpool->queue_not_full),
                                     &(tpool->queue_lock))) != 0) {
            logFmsg(2, "thrFadd_work: cond_wait failed. (e: %d)", ret);
            exit(1);
        }
    }

    if (tpool->shutdown || tpool->queue_closed) {
        if ((ret = pthread_mutex_unlock(&(tpool->queue_lock))) != 0) {
            logFmsg(2, "thrFadd_work: mutex unlock failed. (e: %d)", ret);
            exit(1);
        }
        return -1;
    }
    if ((workp =
         (thrTtpool_work *) malloc(sizeof(thrTtpool_work))) == NULL) {
        logFmsg(2, "thrFadd_work: malloc() failed.");
        exit(1);
    }
    workp->routine = routine;
    workp->arg = arg;
    workp->next = NULL;

    if (tpool->cur_queue_size == 0) {
        tpool->cur_queue_size++;

        tpool->queue_tail = tpool->queue_head = workp;

        if ((ret = pthread_cond_signal(&(tpool->queue_not_empty))) != 0) {
            logFmsg(2,
                    "thrFadd_work: conditional signal on pool failed. (e: %d)",
                    ret);
            exit(1);
        }
    } else {
        tpool->cur_queue_size++;

        tpool->queue_tail->next = workp;
        tpool->queue_tail = workp;

        if ((ret = pthread_cond_signal(&(tpool->queue_not_empty))) != 0) {
            logFmsg(2,
                    "thrFadd_work: conditional signal on not empty failed. (e: %d)",
                    ret);
            exit(1);
        }
    }

    if ((ret = pthread_mutex_unlock(&(tpool->queue_lock))) != 0) {
        logFmsg(2, "thrFadd_work: mutex unlock on queue failed. (e: %d)",
                ret);
        exit(1);
    }
    /*
     * Check to see if all the threads are currently in use, and if we've
     * started them all or not. If we haven't, we're going to go ahead and
     * start another one now. This is coded like this so that the mutex isn't
     * locked every time work is added, rather, only when it's likely that
     * another thread is needed. Synchronization on demand. Once all the
     * threads are started, we won't check at all.
     */

    if (tpool->started < tpool->num_threads) {
        if ((tpool->started - tpool->inuse) <= 5) {
            pthread_mutex_lock(&tpool->thread_lock);
            if (tpool->started < tpool->num_threads) {
                thrTinfo *info = (thrTinfo *) malloc(sizeof(thrTinfo));
                pthread_t tid;

                info->index = tpool->started - 1;
                info->tpool = tpool;

                /*
                 * logFmsg (3, "thrFadd_work: starting new worker thread.
                 * (sid: %d)", info->index);
                 */

                if ((ret =
                     pthread_create((pthread_t *) & tid, NULL, thrFthread,
                                    (void *) info)) != 0) {
                    logFmsg(2,
                            "thrFinit: Unable to create first pool thread. (e: %d)",
                            ret);
                    exit(1);
                }
                tpool->started++;
            }
            pthread_mutex_unlock(&tpool->thread_lock);
        }
    }
    return 1;
}

int thrFdestroy(thrTtpool * tpool, int finish)
{
    int i, ret;
    thrTtpool_work *cur_nodep;

    if ((ret = pthread_mutex_lock(&(tpool->queue_lock))) != 0)
        fprintf(stderr, "pthread_mutex_lock %d", ret), exit(1);

    /* Is a shutdown already in progress? */
    if (tpool->queue_closed || tpool->shutdown) {
        if ((ret = pthread_mutex_unlock(&(tpool->queue_lock))) != 0)
            fprintf(stderr, "pthread_mutex_unlock %d", ret), exit(1);
        return 0;
    }
    tpool->queue_closed = 1;

    if (finish == 1) {
        while (tpool->cur_queue_size != 0) {
            if ((ret = pthread_cond_wait(&(tpool->queue_empty),
                                         &(tpool->queue_lock))) != 0)
                fprintf(stderr, "pthread_cond_wait %d", ret), exit(1);
        }
    }
    tpool->shutdown = 1;

    if ((ret = pthread_mutex_unlock(&(tpool->queue_lock))) != 0)
        fprintf(stderr, "pthread_mutex_unlock %d", ret), exit(1);


    if ((ret = pthread_cond_signal(&(tpool->queue_not_empty))) != 0)
        fprintf(stderr, "pthread_cond_signal %d", ret), exit(1);
    if ((ret = pthread_cond_broadcast(&(tpool->queue_not_full))) != 0)
        fprintf(stderr, "pthread_cond_broadcast %d", ret), exit(1);


    for (i = 0; i < tpool->num_threads; i++) {
        if ((ret = pthread_cancel(tpool->threads[i])) != 0)
            logFmsg(2, "tpool: pthread_cancel (e:%d)", ret);

        if ((ret = pthread_join(tpool->threads[i], NULL)) != 0)
            logFmsg(2, "tpool: pthread_join (e:%d)", ret);
    }

    free(tpool->threads);
    while (tpool->queue_head != NULL) {
        cur_nodep = tpool->queue_head->next;
        tpool->queue_head = tpool->queue_head->next;
        free(cur_nodep);
    }
    free(tpool);

    return 1;
}

void *thrFthread(void *arg)
{
    thrTinfo *info = (thrTinfo *) arg;
    int ret;
    thrTtpool_work *my_workp;

    info->tpool->threads[info->index] = pthread_self();
    info->tpool->trigger[info->index].tid = pthread_self();
    if (pthread_setspecific
        (thrKtrigger, &info->tpool->trigger[info->index])
        != 0) {
        logFmsg(2, "thrFthread: pthread_setspecific() failed. e: %d",
                errno);
        logFmsg(3, "thrFthread: Failed to register I/O trigger.");
        exit(0);
    }
    while (1) {
        if ((ret = pthread_mutex_lock(&(info->tpool->queue_lock))) != 0) {
            logFmsg(2, "tpool: Mutex lock returned error. (e: %d)", ret);
        }
        while ((info->tpool->cur_queue_size == 0)
               && (!info->tpool->shutdown)) {
            if ((ret =
                 pthread_cond_wait(&(info->tpool->queue_not_empty),
                                   &(info->tpool->queue_lock))) != 0) {
                logFmsg(2, "tpool: cond_wait returned fatally. (e: %d)",
                        ret);
                exit(1);
            }
        }

        info->tpool->inuse++;

        if (info->tpool->shutdown == 1) {
            if ((ret =
                 pthread_mutex_unlock(&(info->tpool->queue_lock))) != 0) {
                logFmsg(2, "tpool: pthread_mutex_unlock (e: %d)", ret);
                exit(1);
            }
            free(info);
            pthread_exit(NULL);
        }
        my_workp = info->tpool->queue_head;
        info->tpool->cur_queue_size--;

        if (info->tpool->cur_queue_size == 0) {
            info->tpool->queue_head = info->tpool->queue_tail = NULL;
        } else {
            info->tpool->queue_head = my_workp->next;
        }

        /* Handle waiting add_work threads */
        if ((!info->tpool->blocking_mode) &&
            (info->tpool->cur_queue_size ==
             (info->tpool->max_queue_size - 1))) {
            if ((ret =
                 pthread_cond_broadcast(&(info->tpool->queue_not_full))) !=
                0) {
                logFmsg(2, "itpool: pthread_cond_broadcast (e: %d)", ret);
                exit(1);
            }
        }
        /* Handle waiting destroyer threads */
        if (info->tpool->cur_queue_size == 0) {
            if ((ret =
                 pthread_cond_signal(&(info->tpool->queue_empty))) != 0) {
                logFmsg(2, "pthread_cond_signal (e:%d)", ret);
                exit(1);
            }
        }
        if ((ret = pthread_mutex_unlock(&(info->tpool->queue_lock))) != 0) {
            logFmsg(2, "tpool: pthread_mutex_unlock (e: %d)", ret);
            exit(1);
        }
        (*(my_workp->routine)) (my_workp->arg);
        free(my_workp);
        info->tpool->inuse--;
    }
}
