/*
 * $Header: /datum01/src/continuity/src/dispatch.c,v 1.78.2.5 2002/09/15
 * 02:32:40 aleigh Exp $
 */

/*
 * Copyright (c) 1994, 2004 Alex Leigh, All Rights Reserved
 * 
 * THIS IS UNPUBLISHED PROPRIETARY SOURCE CODE OF ALEX LEIGH The copyright
 * notice above does not evidence any actual or intended publication of such
 * source code.
 * 
 * Disclosure, copying, distribution, or transmission of this code without
 * express permission is prohibited.
 */

#undef POLL_LISTENER

/*
 * The new role of the dispatcher in the v3 configuration is to monitor a
 * network socket for new connections. When a connection is received, a pool
 * thread is used to run a user-provided function to handle the request in
 * whatever manner is appropriate.
 */

#include "csys.h"
#include "mecha.h"

struct disSsock {
    int sd;
    uint32_t cli_ipv4_addr;
};
typedef struct disSsock disTsock;

struct disSlistener {
    int sd;
    int ipv4_port;
    unsigned int ipv4_ip;
    const char *socketstash_id;
    int thr_max;
    int thr_runq;
    int thr_block;
    const char *handler_name;
    int (*handler)(void *, lstTset *);
    thrTtpool *req_pool;
};

#include "continuity.h"

int disFwait_for_startup(disTlistener * listener)
{
    conFwait_for_init();

    /* We have to create this here, rather than in disFsetup(), because on Linux setguid()
     * does not set the user-id for all the threads of a process.
     */

    listener->req_pool =
        thrFinit(listener->thr_max, listener->thr_runq,
                 listener->thr_block);

    return disFlisten_loop(listener);
}

int ipv4_openport(MECHA_UNUSED_ARGUMENT void *p, lstTset * opts)
{
    const char *port;
    const char *ip;
    const char *id;
    unsigned int port_int;
    unsigned int ip_int;
    int sd;

    port = lstFset_get(opts, "port");
    if (port == NULL) {
        logFmsg(2,"port parameter unset!");
        exit(5);
    }

    port_int=atoi(port);

    id = lstFset_get(opts,"id");
    if(id==NULL) {
        logFmsg(2,"id parameter unset!");
        exit(5);
    }

    ip = lstFset_get(opts, "ip");
    if (ip != NULL) {
        struct in_addr saddr;
        
        if (inet_aton(ip, &saddr) == 0) {
            logFmsg(2, "Invalid IP specified.");
            return STATUS_ERROR;
        }
        
        ip_int = ntohl(saddr.s_addr);
    } else {
        ip_int = 0;
    }

    sd = netFipv4_listen_tcp(port_int,ip_int, 5);

    if(sd<0) {
        logFmsg(2,"Unable to open socket. e: %d", errno);
        exit(5);
    }

    conFsocketstash_update(sd,atoi(id));

    return STATUS_OK;
}

int ipv4_dispatcher(MECHA_UNUSED_ARGUMENT void *p, lstTset * opts)
{
    const char *port;
    const char *ip;
    const char *threads = lstFset_get(opts, "threads");
    const char *queue = lstFset_get(opts, "queue");
    const char *id = lstFset_get(opts,"id");

    disTlistener *listener;
    pthread_t tid;

    listener = (disTlistener *) malloc(sizeof(disTlistener));
    bzero(listener, sizeof(disTlistener));

    listener->socketstash_id=id;

    listener->handler_name = lstFset_get(opts, "handler");
    if (listener->handler_name == NULL) {
        logFmsg(2, "Handler not specified.");
        return STATUS_ERROR;
    }
    listener->handler = (cont_handler)dlFget_addr(listener->handler_name);
    if (listener->handler == NULL) {
        logFmsg(2, "Unable to map handler function.");
        return STATUS_ERROR;
    }
    port = lstFset_get(opts, "port");
    if (port == NULL) {
        logFmsg(3, "Port unspecified.");
        return STATUS_ERROR;
    }
    listener->ipv4_port = atoi(port);

    if (threads == NULL)
        listener->thr_max = 32;
    else
        listener->thr_max = atoi(threads);

    logFmsg(3, "pool threads: %d", listener->thr_max);

    if (queue == NULL)
        listener->thr_runq = 512;
    else
        listener->thr_runq = atoi(queue);

    listener->thr_block = 0;

    ip = lstFset_get(opts, "ip");
    if (ip != NULL) {
        struct in_addr saddr;

        if (inet_aton(ip, &saddr) == 0) {
            logFmsg(2, "Invalid IP specified.");
            return STATUS_ERROR;
        }

        listener->ipv4_ip = ntohl(saddr.s_addr);
    } else {
        listener->ipv4_ip = 0;
    }

    listener->sd = disFsetup(listener);

    if (listener->sd < 0)
        return STATUS_ERROR;

    pthread_create(&tid, NULL, (void *(*)(void *)) disFwait_for_startup,
                   listener);

    return STATUS_OK;
}

/*
 * Create a dispatcher.
 */
int disFsetup(disTlistener * listener)
{
    int sock;

    logFmsgid(0, "CON", 10, listener->thr_max, listener->thr_runq,
              listener->thr_block);

    if(listener->socketstash_id!=NULL) {
        logFmsg(3,"Getting socket from stash.");
        sock=conFsocketstash_find(atoi(listener->socketstash_id));
        logFmsg(3,"socket is %d", sock);
    } else {
        sock = netFipv4_listen_tcp(listener->ipv4_port, listener->ipv4_ip, 5);
    }

    if (sock < 0) {
        logFmsgid(2, "CON", 5, errno);
        exit(5);
    }

    fleFset_close_exec(sock);

    return sock;
}

#ifndef POLL_LISTENER
int disFlisten_loop(disTlistener * listener)
{
    struct sockaddr_in ca;

    int calen = sizeof(ca);
    int ns;
    struct in_addr *ia;
    pthread_attr_t tattr;
    netTconn *conn;

    assert(listener->handler != NULL);
    assert(listener->req_pool != NULL);

    listen(listener->sd, 1024);

    logFmsgid(0, "CON", 6, pthread_self(), listener->handler);

    pthread_attr_init(&tattr);
    pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED);

    while (1) {
        conn = (netTconn *) malloc(sizeof(netTconn));

        ns = accept(listener->sd, (struct sockaddr *) &ca, (socklen_t *) &calen);

        if (ns == -1) {
            if (errno == EINTR) {
                logFmsgid(0, "CON", 7);
                intr_handler(0);
            }
            logFmsgid(1, "CON", 8, strerror(errno));
            continue;
        }

        fleFset_close_exec(ns);

        netFset_reuse(ns);      /* Set the socket to be reusable */
/*      netFset_nodelay(ns); */

#ifdef NOTDEF
        /* Set the socket to nonblocking */
        if ((fd_flags = fcntl(ns, F_GETFL)) == -1)
            return -1;
        fd_flags |= O_NONBLOCK;
        if (fcntl(ns, F_SETFL, fd_flags) == -1)
            return -1;
#endif

        ia = &ca.sin_addr;

        conn->sd = ns;
        conn->cli_ipv4_addr = ntohl(ia->s_addr);
        conn->io_read = &netFconn_os_read;
        conn->io_write = &netFconn_os_write;
        conn->io_close = &netFconn_os_close;
        conn->io_writev = &netFconn_os_writev;
        conn->data_free = NULL;

        tasFstat_counter_add("net", 0, "conn_rcvd", 1);

        /*
         * It's up to the work fucntion to free the connection structure
         * whenever it's done with it
         */

        if (thrFadd_work(listener->req_pool, (mecha_handler)listener->handler, conn) == -1) {
            logFmsgid(1, "CON", 9);
            conn->io_close(conn);
        }
    }
}

#else
/*
 * this is the primary function that the dispatcher spins in while receiving
 * connections. A connection is received and then placed into the poll list.
 * If it becomes available for read, it is put on the work list where a
 * thread will pick it up when one becomes available. This ensures that the
 * request data has been received, which prevents read() from spinning on the
 * socket later.
 */

int disFlisten_loop(disTlistener * listener)
{
    struct sockaddr_in ca;
    int calen = sizeof(ca);
    int ns, fd_flags;
    struct in_addr *ia;
    pthread_attr_t tattr;
    netTconn *conn;
    int i, ret;

    struct pollfd *pfds;
    int nfds;
    disTsock *socks;
    int maxsds = listener->thr_max + 1;

    assert(listener->handler != NULL);
    assert(listener->req_pool != NULL);

    listen(listener->sd, 1024);

    logFmsgid(0, "CON", 6, pthread_self(), listener->handler);

    pthread_attr_init(&tattr);
    pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED);

    /*
     * Create a pfd array large enough to hold an entry for every possible
     * connection, which is equal to the number of threads.
     */
    pfds = (struct pollfd *) malloc(sizeof(struct pollfd) * maxsds);

    socks = (disTsock *) malloc(sizeof(disTsock) * maxsds);
    bzero(socks, sizeof(disTsock) * maxsds);

    if ((fd_flags = fcntl(listener->sd, F_GETFL)) == -1)
        return -1;
    fd_flags |= O_NONBLOCK;
    if (fcntl(listener->sd, F_SETFL, fd_flags) == -1)
        return -1;

    pfds[0].fd = listener->sd;
    pfds[0].events = POLLIN | POLLPRI;

    while (1) {
        /* Create poll list of socks currently in socks */

        /* { utlTswatch *sw = utlFstart_swatch(); */
        nfds = 1;               /* we start with the listener */
        for (i = 0; i < listener->thr_max; i++) {
            if (socks[i].sd > 0) {
                pfds[nfds].fd = socks[i].sd;
                pfds[nfds].events = POLLIN | POLLPRI;
                nfds++;
            }
        }
        /* logFmsg(3,"poll build took %d us", utlFstop_swatch(sw));} */


        /* { utlTswatch *sw = utlFstart_swatch(); */
        ret = poll(pfds, nfds, 5000);
        /* logFmsg(3,"poll() took %d us", utlFstop_swatch(sw));} */


        if (ret == -1) {
            logFmsg(2, "poll() returned -1 (e: %d)", errno);
            exit(1);
        }
        if (poll > 0) {
            for (i = 1; i < nfds; i++) {
                if ((pfds[i].revents & POLLIN) == POLLIN) {
                    int x;

                    for (x = 0; x < listener->thr_max; x++) {
                        if (socks[x].sd == pfds[i].fd) {
                            conn = (netTconn *) malloc(sizeof(netTconn));

                            conn->cli_ipv4_addr = socks[x].cli_ipv4_addr;
                            conn->sd = socks[x].sd;
                            conn->io_read = &netFconn_os_read;
                            conn->io_write = &netFconn_os_write;
                            conn->io_close = &netFconn_os_close;
                            conn->io_writev = &netFconn_os_writev;
                            conn->data_free = NULL;


                            logFmsg(3, "%d", conn->sd);

                            socks[x].sd = 0;

                            tasFstat_counter_add("net", 0, "conn_rcvd", 1);

                            if (thrFadd_work
                                (listener->req_pool, listener->handler,
                                 conn) == -1) {
                                logFmsgid(1, "CON", 9);
                                close(conn->sd);
                                free(conn);
                            }
                        }
                    }
                }
            }

            /* if (pfds[0].revents & POLLIN == POLLIN) { */
            ns = accept(listener->sd, (struct sockaddr *) &ca,
                        (int *) &calen);

            if (ns == -1) {
                if (errno == EINTR) {
                    logFmsgid(0, "CON", 7);
                    intr_handler(0);
                }
                /* logFmsgid(1, "CON", 8, strerror(errno)); */
                continue;
            }
            netFset_reuse(ns);  /* Set the socket to be reusable */
            netFset_nodelay(ns);        /* Disable the NAGLE algorithm on the socket */

            {
                unsigned int x = 50000;
                if (setsockopt
                    (ns, SOL_SOCKET, SO_SNDBUF, &x,
                     sizeof(unsigned int)) == -1) {
                    logFmsg(3,
                            "setsockopt() SNDBUF failed: e: %d",
                            errno);
                }
            }

            /* Set the socket to nonblocking */
            if ((fd_flags = fcntl(ns, F_GETFL)) == -1)
                return -1;
            fd_flags |= O_NONBLOCK;
            if (fcntl(ns, F_SETFL, fd_flags) == -1)
                return -1;

            ia = &ca.sin_addr;
            for (i = 0; i < listener->thr_max; i++) {
                if (socks[i].sd == 0) {
                    socks[i].sd = ns;
                    socks[i].cli_ipv4_addr = ntohl(ia->s_addr);
                    break;
                }
            }
        }
    }
}

#endif
