ocs_ramd_thread.c

Go to the documentation of this file.

/*
 * Copyright (c) 2011-2015, Emulex
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 */

/**
 * @file
 *
 */

#include "ocs.h"
#include "ocs_ramd.h"
#include "ocs_ramd_thread.h"

static uint32_t threads_per_cpu = 2;
static ocs_array_t *ramd_cmd_array;     /* cmd array for ramd cmd queues */
static ocs_cbuf_t *ramd_cmd_free_queue;     /* cmd free list */
static ocs_atomic_t *ramd_threads_next;     /* next thread within a cpu */
static ocs_ramd_thread_t **ramd_threads;    /* ramd_threads */
static uint32_t enable_ramd_threading;
static uint32_t thread_cmds;

static int32_t
ocs_ramd_server_thread(ocs_thread_t *mythread)
{
    int32_t rc;
    ocs_ramd_thread_t *thr = mythread->arg;
    ocs_ramd_cmd_t *cmd;

    ocs_log_debug(NULL, "%s: ramd running %p\n", mythread->name, thr->cmd_queue);
    while (!ocs_thread_terminate_requested(mythread)) {
        cmd = ocs_cbuf_get(thr->cmd_queue, 100000);
        if (cmd != NULL) {
            OCS_STAT(thr->cmd_count[cmd->ramd_cmd_type]++);
            switch(cmd->ramd_cmd_type) {
            case OCS_RAMD_CMD_RECV_CMD:
                rc = _ocs_scsi_recv_cmd(cmd->ramd_cmd.recv_cmd.io,
                            cmd->ramd_cmd.recv_cmd.lun,
                            cmd->ramd_cmd.recv_cmd.cdb,
                            cmd->ramd_cmd.recv_cmd.cdb_len,
                            cmd->ramd_cmd.recv_cmd.flags);
                if (rc) {
                    ocs_log_test(NULL, "ocs_scsi_recv_cmd_common returned %d\n", rc);
                }
                break;
            case OCS_RAMD_CMD_DATAPHASE_CB:
                rc = _scsi_dataphase_cb(cmd->ramd_cmd.dataphase_cb.io,
                            cmd->ramd_cmd.dataphase_cb.scsi_status,
                            cmd->ramd_cmd.dataphase_cb.flags,
                            cmd->ramd_cmd.dataphase_cb.arg);
                if (rc) {
                    ocs_log_test(NULL, "_scsi_dataphase_cb returned %d\n", rc);
                }
                break;
            case OCS_RAMD_CMD_IO_CB:
                rc = _scsi_io_cb(cmd->ramd_cmd.io_cb.io,
                         cmd->ramd_cmd.io_cb.scsi_status,
                         cmd->ramd_cmd.io_cb.flags,
                         cmd->ramd_cmd.io_cb.arg);
                if (rc) {
                    ocs_log_test(NULL, "_scsi_io_cb returned %d\n", rc);
                }
                break;
            case OCS_RAMD_CMD_BOUNCE:
                (*cmd->ramd_cmd.bounce.fctn)(cmd->ramd_cmd.bounce.arg);
                break;
            default:
                break;
            }
            ocs_cbuf_put(ramd_cmd_free_queue, cmd);
        }
    }
    return 0;
}

int32_t
ocs_ramd_start_threads(uint32_t threading_mode, uint32_t arg_thread_cmds)
{
    int32_t rc;
    uint32_t i;
    uint32_t num_cpus = ocs_get_num_cpus();
    uint32_t cpuidx;
    uint32_t thridx;
    ocs_ramd_thread_t *pthr;

    enable_ramd_threading = threading_mode;
    thread_cmds = arg_thread_cmds;

    /* Don't start threads if not enabled */
    if (enable_ramd_threading) {

        /* Create the free queue */
        ramd_cmd_free_queue = ocs_cbuf_alloc(NULL, 1000);
        if (ramd_cmd_free_queue == NULL) {
            ocs_log_err(NULL, "ocs_cbuf_alloc cmd_free_queue failed\n");
            ocs_ramd_stop_threads();
            return -1;
        }
        ramd_cmd_array = ocs_array_alloc(NULL, sizeof(ocs_ramd_cmd_t), 1000);
        ocs_cbuf_prime(ramd_cmd_free_queue, ramd_cmd_array);

        /* Allocate 2D array of threads */
        ramd_threads = ocs_malloc(NULL, num_cpus * sizeof(ocs_ramd_thread_t*), OCS_M_ZERO | OCS_M_NOWAIT);
        for (i = 0; i < num_cpus; i++) {
            ramd_threads[i] = ocs_malloc(NULL, threads_per_cpu * sizeof(ocs_ramd_thread_t), OCS_M_ZERO | OCS_M_NOWAIT);
        }
        ramd_threads_next = ocs_malloc(NULL, num_cpus * sizeof(*ramd_threads_next), OCS_M_ZERO | OCS_M_NOWAIT);
        for (i = 0; i < num_cpus; i++) {
            ocs_atomic_init(&ramd_threads_next[i], 0);
        }

        for (cpuidx = 0; cpuidx < num_cpus; cpuidx++) {
            for (thridx = 0; thridx < threads_per_cpu; thridx++) {
                char name[64];
                pthr = &ramd_threads[cpuidx][thridx];

                /* Create command queue */
                pthr->cmd_queue = ocs_cbuf_alloc(NULL, 100);
                if (pthr->cmd_queue == NULL) {
                    ocs_log_err(NULL, "ocs_cbuf_alloc failed\n");
                    ocs_ramd_stop_threads();
                    return -1;
                }

                /* Create thread */
                ocs_snprintf(name, sizeof(name), "ocs_ramd:%d:%d", cpuidx, thridx);

                rc = ocs_thread_create(NULL, &pthr->thread, ocs_ramd_server_thread, name,
                               pthr, OCS_THREAD_CREATE);
                if (rc) {
                    ocs_log_err(NULL, "ocs_thread_create failed: %d\n", rc);
                    ocs_ramd_stop_threads();
                    return -1;
                }

                /* Set priority */

                /* Set CPU affinity ... */
                rc = ocs_thread_setcpu(&pthr->thread, cpuidx);
                if (rc == 0) {
                    ocs_log_debug(NULL, "thread %s locked to cpu %d\n", pthr->thread.name,
                        pthr->thread.cpu_affinity);
                }
            }
        }

        for (cpuidx = 0; cpuidx < num_cpus; cpuidx++) {
            for (thridx = 0; thridx < threads_per_cpu; thridx++) {
                /* Start */
                pthr = &ramd_threads[cpuidx][thridx];
                pthr->started = TRUE;
                ocs_thread_start(&pthr->thread);
            }
        }
    }
    return 0;
}

void
ocs_ramd_stop_threads(void)
{
    uint32_t num_cpus = ocs_get_num_cpus();
    uint32_t cpuidx;
    uint32_t thridx;
    int32_t rc;
    ocs_ramd_thread_t *pthr;

    if (enable_ramd_threading) {

        for (cpuidx = 0; cpuidx < num_cpus; cpuidx++) {
            for (thridx = 0; thridx < threads_per_cpu; thridx++) {
                pthr = &ramd_threads[cpuidx][thridx];
                if (pthr->started) {
                    rc = ocs_thread_terminate(&pthr->thread);
                    if (rc) {
                        ocs_log_test(NULL, "ocs_thread_terminate failed: %d\n", rc);
                    }
                    pthr->started = FALSE;
                }
                ocs_cbuf_free(pthr->cmd_queue);
            }
            ocs_free(NULL, ramd_threads[cpuidx], threads_per_cpu*sizeof(ocs_ramd_thread_t));
        }

        ocs_free(NULL, ramd_threads, num_cpus * sizeof(*ramd_threads));
        ocs_free(NULL, ramd_threads_next, num_cpus * sizeof(*ramd_threads_next));
        ocs_array_free(ramd_cmd_array);
        ocs_cbuf_free(ramd_cmd_free_queue);
    }
}

void
ocs_ramd_thread_ddump(ocs_textbuf_t *textbuf)
{
#if OCS_STAT_ENABLE
    uint32_t num_cpus = ocs_get_num_cpus();
    uint32_t cpuidx;
    uint32_t thridx;
    uint32_t instance;
    uint32_t cmdidx;
    ocs_ramd_thread_t *pthr;

    if (ramd_threads != NULL) {
        ocs_ddump_section(textbuf, "ramd_threads", 0);
        for (instance = 0., cpuidx = 0; cpuidx < num_cpus; cpuidx++) {
            for (thridx = 0; thridx < threads_per_cpu; thridx++) {
                pthr = &ramd_threads[cpuidx][thridx];
                ocs_ddump_section(textbuf, "ramd_thread", instance);
                for (cmdidx = 0; cmdidx < OCS_RAMD_CMD_MAX; cmdidx++) {
                    switch(cmdidx) {
                    case OCS_RAMD_CMD_RECV_CMD:
                        ocs_ddump_value(textbuf, "recv_cmd", "%d %d", cmdidx,
                                pthr->cmd_count[cmdidx]);
                        break;
                    case OCS_RAMD_CMD_DATAPHASE_CB:
                        ocs_ddump_value(textbuf, "dataphase_cb", "%d %d", cmdidx,
                                pthr->cmd_count[cmdidx]);
                        break;
                    case OCS_RAMD_CMD_IO_CB:
                        ocs_ddump_value(textbuf, "io_cb", "%d %d", cmdidx,
                                pthr->cmd_count[cmdidx]);
                        break;
                    default:
                        break;
                    }
                }
                ocs_ddump_endsection(textbuf, "ramd_thread", instance);
                instance++;
            }
        }
        ocs_ddump_endsection(textbuf, "ramd_threads", 0);
    }
#endif
}

uint32_t
ocs_ramd_get_enable_ramd_threading(void)
{
    return enable_ramd_threading;
}

int32_t
ocs_scsi_recv_cmd(ocs_io_t *io, uint32_t lun, uint8_t *cdb, uint32_t cdb_len, uint32_t flags)
{
    ocs_ramd_cmd_t *cmd;
    ocs_ramd_thread_t *pthr;
    int32_t rc;
    uint32_t num_cpus = ocs_get_num_cpus();
    uint32_t cpuidx;
    uint32_t idx;

    if (thread_cmds == 0) {
        return _ocs_scsi_recv_cmd(io, lun, cdb, cdb_len, flags);
    }

    cmd = ocs_cbuf_get(ramd_cmd_free_queue, -1);
    if (cmd == NULL) {
        ocs_log_test(io->ocs, "ocs_cbuf_get() free queue failed\n");
        return -1;
    }

    cmd->ramd_cmd_type = OCS_RAMD_CMD_RECV_CMD;
    cmd->ramd_cmd.recv_cmd.io = io;
    cmd->ramd_cmd.recv_cmd.lun = lun;
    ocs_assert(cdb_len < sizeof(cmd->ramd_cmd.recv_cmd.cdb), -1);
    ocs_memcpy(cmd->ramd_cmd.recv_cmd.cdb, cdb, cdb_len);
    cmd->ramd_cmd.recv_cmd.cdb_len = cdb_len;
    cmd->ramd_cmd.recv_cmd.flags = flags;

    /* Pick a thread and pass the command to it */
    cpuidx = io->init_task_tag % num_cpus;
    idx = ocs_atomic_add_return(&ramd_threads_next[cpuidx], 1) % threads_per_cpu;

    pthr = &ramd_threads[cpuidx][idx];

    /* save thread pointer for future reference */
    io->tgt_io.thr = pthr;

    rc = ocs_cbuf_put(pthr->cmd_queue, cmd);
    if (rc < 0) {
        ocs_log_test(io->ocs, "ocs_cbuf_put() failed: %d\n", rc);
        ocs_cbuf_put(ramd_cmd_free_queue, cmd);
        return -1;
    }

    return 0;
}

int32_t
scsi_dataphase_cb(ocs_io_t *io, ocs_scsi_io_status_e scsi_status, uint32_t flags, void *arg)
{
    ocs_ramd_cmd_t *cmd;
    ocs_ramd_thread_t *pthr;
    int32_t rc;

    if (io->tgt_io.thr == NULL) {
        return _scsi_dataphase_cb(io, scsi_status, flags, arg);
    }

    cmd = ocs_cbuf_get(ramd_cmd_free_queue, -1);
    if (cmd == NULL) {
        ocs_log_test(io->ocs, "ocs_cbuf_get() free queue failed\n");
        return -1;
    }

    cmd->ramd_cmd_type = OCS_RAMD_CMD_DATAPHASE_CB;
    cmd->ramd_cmd.dataphase_cb.io = io;
    cmd->ramd_cmd.dataphase_cb.scsi_status = scsi_status;
    cmd->ramd_cmd.dataphase_cb.flags = flags;
    cmd->ramd_cmd.dataphase_cb.arg = arg;

    pthr = io->tgt_io.thr;

    rc = ocs_cbuf_put(pthr->cmd_queue, cmd);
    if (rc < 0) {
        ocs_log_test(io->ocs, "ocs_cbuf_put failed: %d\n", rc);
        ocs_cbuf_put(ramd_cmd_free_queue, cmd);
        return -1;
    }
    return 0;
}

int32_t
scsi_io_cb(ocs_io_t *io, ocs_scsi_io_status_e scsi_status, uint32_t flags, void *arg)
{
    ocs_ramd_cmd_t *cmd;
    ocs_ramd_thread_t *pthr;
    int32_t rc;

    if (io->tgt_io.thr == NULL) {
        return _scsi_io_cb(io, scsi_status, flags, arg);
    }

    cmd = ocs_cbuf_get(ramd_cmd_free_queue, -1);
    if (cmd == NULL) {
        ocs_log_test(io->ocs, "ocs_cbuf_get() free queue failed\n");
        return -1;
    }

    cmd->ramd_cmd_type = OCS_RAMD_CMD_IO_CB;
    cmd->ramd_cmd.dataphase_cb.io = io;
    cmd->ramd_cmd.dataphase_cb.scsi_status = scsi_status;
    cmd->ramd_cmd.dataphase_cb.flags = flags;
    cmd->ramd_cmd.dataphase_cb.arg = arg;

    pthr = io->tgt_io.thr;

    rc = ocs_cbuf_put(pthr->cmd_queue, cmd);
    if (rc < 0) {
        ocs_log_test(io->ocs, "ocs_cbuf_put failed: %d\n", rc);
        ocs_cbuf_put(ramd_cmd_free_queue, cmd);
        return -1;
    }
    return 0;
}

void
ocs_ramd_tgt_bounce(void (*fctn)(void *arg), void *arg, uint32_t s_id, uint32_t d_id, uint32_t ox_id)
{
    ocs_ramd_cmd_t *cmd;
    ocs_ramd_thread_t *pthr;
    uint32_t idx;
    int32_t rc;
    uint32_t cpu;

    /* If threading is not enabled, then just make the callback */
    if (!ocs_ramd_get_enable_ramd_threading()) {
        (*fctn)(arg);
    } else {
        cpu = ox_id % ocs_get_num_cpus();

        cmd = ocs_cbuf_get(ramd_cmd_free_queue, -1);
        if (cmd == NULL) {
            ocs_log_test(NULL, "ocs_cbuf_get() free queue failed\n");
            return;
        }

        cmd->ramd_cmd_type = OCS_RAMD_CMD_BOUNCE;
        cmd->ramd_cmd.bounce.fctn = fctn;
        cmd->ramd_cmd.bounce.arg = arg;

        idx = ocs_atomic_add_return(&ramd_threads_next[cpu], 1) % threads_per_cpu;
        pthr = &ramd_threads[cpu][idx];

        rc = ocs_cbuf_put(pthr->cmd_queue, cmd);
        if (rc < 0) {
            ocs_log_test(NULL, "ocs_cbuf_put failed: %d\n", rc);
            ocs_cbuf_put(ramd_cmd_free_queue, cmd);
            return;
        }
    }
}