ocs_recovery.c

Go to the documentation of this file.

/*
 * Copyright (c) 2011-2015, Emulex
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include "ocs.h"
#include "ocs_recovery.h"

/* Define FW error recovery context */
ocs_fw_error_recovery_t ocs_recovery;

/**
 * @brief Handle the EQ processing for all the ocs instances of the adapter
 */
static void ocs_adapter_event_processing(ocs_t *ocs, bool port_reset, bool start)
{
#if !defined(OCSU_FC_RAMD) && !defined(OCSU_FC_WORKLOAD) && !defined(OCS_USPACE_SPDK)
    ocs_t *other_ocs = NULL;
    uint8_t bus, dev, func;
    uint8_t other_bus, other_dev, other_func;
    uint32_t index = 0;

    ocs_get_bus_dev_func(ocs, &bus, &dev, &func);
    while ((other_ocs = ocs_get_instance(index++)) != NULL) {
        ocs_get_bus_dev_func(other_ocs, &other_bus, &other_dev, &other_func);
        if ((bus == other_bus) && (dev == other_dev)) {
            if (port_reset && (func != other_func)) {
                /* For FDD, handle the EQ processing only on the specified pci func */
                continue;
            }

            if (start) {
                ocs_log_info(other_ocs, "Start EQ processing\n");
                ocs_start_event_processing(&other_ocs->ocs_os);
            } else {
                ocs_log_info(other_ocs, "Stop EQ processing\n");
                ocs_stop_event_processing(&other_ocs->ocs_os);
            }
        }
    }
#endif
}

/**
 * @brief Set OCS HAL state to inactive for all the ocs instances of the adapter
 */
static void ocs_adapter_hal_inactive(ocs_t *ocs, bool port_reset)
{
    ocs_t *other_ocs = NULL;
    uint8_t bus, dev, func;
    uint8_t other_bus, other_dev, other_func;
    uint32_t index = 0;

    ocs_get_bus_dev_func(ocs, &bus, &dev, &func);
    while ((other_ocs = ocs_get_instance(index++)) != NULL) {
        ocs_get_bus_dev_func(other_ocs, &other_bus, &other_dev, &other_func);
        if ((bus == other_bus) && (dev == other_dev)) {
            if (port_reset && (func != other_func)) {
                continue;
            }

            if (OCS_HAL_STATE_ACTIVE == other_ocs->hal.state) {
                other_ocs->hal.state = OCS_HAL_STATE_QUEUES_ALLOCATED;
            }
        }
    }
}

/**
 * @brief Shutdown sport(s) for all the ocs instances of the adapter
 */
static void ocs_adapter_sport_shutdown(ocs_t *ocs, bool port_reset)
{
    ocs_t *other_ocs;
    uint8_t bus, dev, func;
    uint8_t other_bus, other_dev, other_func;
    uint32_t index = 0;

    ocs_get_bus_dev_func(ocs, &bus, &dev, &func);
    while ((other_ocs = ocs_get_instance(index++)) != NULL) {
        ocs_get_bus_dev_func(other_ocs, &other_bus, &other_dev, &other_func);
        if ((bus == other_bus) && (dev == other_dev)) {
            if (port_reset && (func != other_func)) {
                continue;
            }

            ocs_scsi_tgt_cancel_io(other_ocs);
#if !defined(OCSU_FC_RAMD) && !defined(OCSU_FC_WORKLOAD) && !defined(OCS_USPACE_SPDK)
            if (other_ocs->domain) {
                ocs_sport_t *sport = NULL;

                ocs_domain_lock(other_ocs->domain);
                ocs_list_foreach(&other_ocs->domain->sport_list, sport) {
                    ocs_sport_nodes_shutdown(other_ocs, sport);
                }
                ocs_domain_unlock(other_ocs->domain);
            }
#endif
        }
    }
}

/**
 * @brief Shutdown xport(s) for all the ocs instances of the adapter
 */
static void ocs_adapter_xport_shutdown(ocs_t *ocs, bool port_reset)
{
    ocs_t *other_ocs;
    uint8_t bus, dev, func;
    uint8_t other_bus, other_dev, other_func;
    uint32_t index = 0;

    ocs_get_bus_dev_func(ocs, &bus, &dev, &func);
    while ((other_ocs = ocs_get_instance(index++)) != NULL) {
        ocs_get_bus_dev_func(other_ocs, &other_bus, &other_dev, &other_func);
        if ((bus == other_bus) && (dev == other_dev)) {
            if (port_reset && (func != other_func)) {
                continue;
            }

            ocs_log_debug(other_ocs, "Initiate transport shutdown\n");
            if (ocs_xport_control(other_ocs->xport, OCS_XPORT_SHUTDOWN)) {
                ocs_log_err(other_ocs, "Tranport shutdown failed\n");
            }
        }
    }
}

/**
 * @brief Bring the links online
 */
static int32_t ocs_adapter_xport_online(ocs_t *ocs, bool port_reset)
{
    ocs_t *other_ocs;
    uint8_t bus, dev, func;
    uint8_t other_bus, other_dev, other_func;
    int32_t rc, ret = 0;
    uint32_t index = 0;

    ocs_get_bus_dev_func(ocs, &bus, &dev, &func);
    while ((other_ocs = ocs_get_instance(index++)) != NULL) {
        ocs_get_bus_dev_func(other_ocs, &other_bus, &other_dev, &other_func);
        if ((bus == other_bus) && (dev == other_dev)) {
            if (port_reset && (func != other_func)) {
                continue;
            }

            rc = ocs_scsi_tgt_port_online(other_ocs);
            if (rc) {
                ocs_log_err(other_ocs, "Unable to bring-up the link\n");
                ret = rc;
            }
        }
    }

    return ret;
}

/**
 * @brief Stop FW error recovery thread
 * 
 * @return void
 */
void ocs_fw_err_recovery_stop(void)
{
    ocs_thread_terminate(&ocs_recovery.thread);
    ocs_lock_free(&ocs_recovery.lock);
    ocs_log_info(NULL, "FW error recovery thread terminated\n");
}

/**
 * @brief Set recovery state
 */
static void ocs_recovery_state_set(uint32_t state)
{
    ocs_lock(&ocs_recovery.lock);
    ocs_recovery.state = state;
    ocs_unlock(&ocs_recovery.lock);
}

/**
 * @brief Check recovery state
 *
 * @return TRUE if the state matches, FALSE otherwise
 */
static int32_t ocs_recovery_state_check(uint32_t state)
{
    int32_t rc = FALSE;

    ocs_lock(&ocs_recovery.lock);
    if (ocs_recovery.state == state) {
        rc = TRUE;
    }
    ocs_unlock(&ocs_recovery.lock);

    return rc;
}

/**
 * @brief Set fW dump state
 *
 * @return void
 */
void ocs_fw_dump_state_set(ocs_t *ocs, uint32_t state)
{
    ocs_t *other_ocs;
    uint8_t bus, dev, func;
    uint8_t other_bus, other_dev, other_func;
    uint32_t index = 0;

    ocs_get_bus_dev_func(ocs, &bus, &dev, &func);
    while ((other_ocs = ocs_get_instance(index++)) != NULL) {
        ocs_get_bus_dev_func(other_ocs, &other_bus, &other_dev, &other_func);
        if ((bus == other_bus) && (dev == other_dev)) {
            if ((FW_CHIP_DUMP_STATE_VALID == state) && (0 != other_func)) {
                /* For CLD, set the dump state only on pci func 0 */
                continue;
            }

            if ((FW_FUNC_DUMP_STATE_VALID == state) && (func != other_func)) {
                /* For FDD, set the dump state only on the specified pci func */
                continue;
            }

            ocs_lock(&other_ocs->fw_dump.lock);
            other_ocs->fw_dump.state = state;
            ocs_unlock(&other_ocs->fw_dump.lock);
        }
    }
}

/**
 * @brief Save dump data to OCS buffer; saved_buff will be copied to the
 *        user space buffer when an application requests for it
 */
static int32_t ocs_fw_dump_save(ocs_t *ocs, uint8_t dump_level)
{
    ocs_t *other_ocs;
    ocs_dma_t *dump_buffers = NULL;
    uint8_t *saved_buff = NULL;
    uint8_t bus, dev, func;
    uint8_t other_bus, other_dev, other_func;
    uint32_t index = 0;
    uint32_t offset = 0;
    uint32_t num_buffers = 0;
    uint32_t saved_buff_size = 0;

    if (FW_FUNC_DESC_DUMP == dump_level) {
        dump_buffers = ocs->fw_dump.func_dump_buffers;
        num_buffers = ocs->fw_dump.num_func_dump_buffers;
        saved_buff = ocs->fw_dump.saved_buff;
        saved_buff_size = ocs->fw_dump.size;
    } else {
        ocs_get_bus_dev_func(ocs, &bus, &dev, &func);
        while ((other_ocs = ocs_get_instance(index++)) != NULL) {
            ocs_get_bus_dev_func(other_ocs, &other_bus, &other_dev, &other_func);
            if ((bus == other_bus) && (dev == other_dev) && (0 == other_func)) {
                dump_buffers = other_ocs->fw_dump.chip_dump_buffers;
                num_buffers = other_ocs->fw_dump.num_chip_dump_buffers;
                saved_buff = other_ocs->fw_dump.saved_buff;
                saved_buff_size = other_ocs->fw_dump.size;
                break;
            }
        }
    }

    if (!dump_buffers || !num_buffers || !saved_buff) {
        ocs_log_err(ocs, "Failed to locate dump buffers\n");
        return -1;
    }

    ocs_memset(saved_buff, 0, saved_buff_size);
    for (index = 0; index < num_buffers; index++) {
        if (ocs_memcpy(saved_buff + offset, dump_buffers[index].virt,
                dump_buffers[index].size) == NULL) {
            ocs_log_err(ocs, "ocs_memcpy() failed\n");
            return -1;
        }

        offset += dump_buffers[index].size;
    }

    ocs_log_info(ocs, "FW dump is saved\n");
    return 0;
}

/**
 * @brief Request FW dump
 *
 * @param ocs Pointer to ocs instance
 * @param dump_level Gendump request level can be either chip-level or fdb
 *
 * @return OCS_FW_DUMP_STATUS_SUCCESS on success,
 *         OCS_FW_DUMP_STATUS_IN_PROGRESS if dump is already in progress, or
 *         OCS_FW_DUMP_STATUS_FAILED on failure.
 */
static int32_t ocs_fw_dump_req(ocs_t *ocs, uint8_t dump_level)
{
    int32_t rc = OCS_FW_DUMP_STATUS_SUCCESS;
    uint32_t skip_dump = FALSE;

    /* Validate SLI pause errors */
    if (sli_is_paused(&ocs->hal.sli)) {
        sli_validate_pause_errors(&ocs->hal.sli, &skip_dump);
        if (skip_dump) {
            /* Don't collect the FW dump */
            dump_level = FW_DUMP_LEVEL_NONE;
        }
    }

    rc = ocs_trigger_dump(ocs, dump_level);
    if (OCS_FW_DUMP_STATUS_SUCCESS == rc) {
        ocs_fw_dump_save(ocs, dump_level);
        ocs_fw_dump_state_set(ocs, dump_level);
    } else {
        ocs_fw_dump_state_set(ocs, FW_DUMP_STATE_NONE);
    }

    return rc;
}

/**
 * @brief Initiate FW dump and do reset
 *
 * @param ocs Pointer to ocs instance
 * @param reset_mode Gendump request mode can be auto-recovery / manual / port-migration
 * @param dump_level Gendump request level can be either chip-level or fdb
 * @param trigger_dump Trigger FW dump
 *
 * @return OCS_RECOVERY_STATUS_SUCCESS on success,
 *         OCS_RECOVERY_STATUS_IN_PROGRESS if recovery is already in progress, or
 *         OCS_RECOVERY_STATUS_FAILED on failure.
 */
int32_t ocs_recovery_reset(ocs_t *ocs, uint32_t reset_mode, uint8_t dump_level, bool trigger_dump)
{
    bool port_reset = FALSE;
    uint8_t bus, dev, func;
    int32_t rc;

    /* Check if recovery state started */
    if (ocs_recovery_state_check(OCS_RECOVERY_STATE_IN_PROGRESS))
        return OCS_RECOVERY_STATUS_IN_PROGRESS;
 
    ocs_recovery_state_set(OCS_RECOVERY_STATE_IN_PROGRESS);
    ocs_get_bus_dev_func(ocs, &bus, &dev, &func);

    /* Check if we have to generate the FW dump */
    if (OCS_RESET_MODE_PORT_MIGRATION != reset_mode) {
        /* Collect the driver dump before triggering the FW dump */
        ocs_adapter_ddump_clear_and_save(ocs, bus, dev,
                OCS_DDUMP_FLAGS_WQES |
                OCS_DDUMP_FLAGS_CQES |
                OCS_DDUMP_FLAGS_MQES, -1, TRUE);

        if (trigger_dump) {
            /* Generate the FW dump only if it's not available */
            rc = ocs_fw_dump_req(ocs, dump_level);
            if (OCS_FW_DUMP_STATUS_IN_PROGRESS == rc) {
                rc = OCS_RECOVERY_STATUS_IN_PROGRESS;
                ocs_log_info(ocs, "FW dump is already in progress\n");
                return rc;
            }

            if (OCS_FW_DUMP_STATUS_FAILED == rc) {
                rc = OCS_RECOVERY_STATUS_FAILED;
                ocs_log_err(ocs, "FW dump has failed\n");
                goto exit_err;
            }
        }

        if (FW_FUNC_DESC_DUMP == dump_level) {
            port_reset = TRUE;
        }
    } else {
        rc = ocs_hal_port_migration(&ocs->hal);
        if (rc) {
            ocs_log_err(ocs, "Port migration has failed\n");
            goto exit_err;
        }
        port_reset = TRUE;
    }

    rc = OCS_RECOVERY_STATUS_SUCCESS;

    /* Wait for all the pending events to be processed */
    ocs_adapter_event_processing(ocs, port_reset, FALSE);

    /* Check and set HAL state inactive so that we don't post any more WQE's */
    ocs_adapter_hal_inactive(ocs, port_reset);

    /* Shutdown sport(s) for all the ocs instances of the adapter */
    ocs_adapter_sport_shutdown(ocs, port_reset);

    /* Shutdown xport(s) on all the ports to remove domain / sport forcefully */
    ocs_adapter_xport_shutdown(ocs, port_reset);

    /* now reset the port / adapter */
    if (port_reset) {
        rc = ocs_port_reset(ocs);
        ocs_log_info(ocs, "OCS port (%02d:%02d:%02d) reset has %s\n",
                bus, dev, func, (rc ? "failed" : "succeeded"));
    } else {
        rc = ocs_adapter_reset(ocs);
        ocs_log_info(ocs, "OCS adapter (%02d:%02d) reset has %s\n",
                bus, dev, (rc ? "failed" : "succeeded"));
    }

    /* Re-enable the interrupts */
    ocs_adapter_event_processing(ocs, port_reset, TRUE);

    /* Bring links on each port to the previous state */
    rc = ocs_adapter_xport_online(ocs, port_reset);
    if (rc) {
        ocs_log_err(ocs, "Failed to bring ports online\n");
    }

exit_err:
    /* Set recovery state to default, irrespective of the rc */
    ocs_recovery_state_set(OCS_RECOVERY_STATE_IDLE);
    return rc;
}

static int32_t ocs_initiate_gendump(ocs_t *ocs, uint8_t dump_level)
{
    int32_t rc = OCS_FW_DUMP_STATUS_SUCCESS;

    /**
     * If FW dump exists already due to recovery processs,
     * return it. Else, do gendump and reset the adapter.
     */
    if (ocs_fw_dump_state_check(ocs, dump_level)) {
        ocs_log_debug(ocs, "FW dump is already present\n");
        return rc;
    }

    rc = ocs_recovery_reset(ocs, OCS_RECOVERY_MODE_MANUAL, dump_level, true);
    if (rc) {
        if (OCS_RECOVERY_STATUS_IN_PROGRESS == rc) {
            ocs_log_debug(ocs, "FW dump is already in progress\n");
            rc = OCS_FW_DUMP_STATUS_IN_PROGRESS;
        } else {
            ocs_log_err(ocs, "FW dump failed\n");
            rc = OCS_FW_DUMP_STATUS_FAILED;
        }
    } else {
        ocs_log_info(ocs, "FW dump is generated\n");
        rc = OCS_FW_DUMP_STATUS_SUCCESS;
    }

    return rc;
}

int32_t ocs_gendump(ocs_t *ocs, uint8_t dump_level)
{
    int32_t rc = OCS_FW_DUMP_STATUS_FAILED;

    if (FW_DUMP_TYPE_FLASH != ocs->fw_dump.type) {
        ocs_log_err(ocs, "Flash gendump is disabled\n");
        return rc;
    }

    rc = ocs_initiate_gendump(ocs, dump_level);
    if (rc) {
        ocs_log_err(ocs, "ocs_initiate_gendump() failed\n");
    }

    return rc;
}

/**
 * @brief Waits for FW_DUMP_PRESENT flag to be set in fw_dump.flags context,
 *    and if recovery is started, waits for recovery completion.
 *
 * @return returns 0 for success, a negative error code value for failure
 */
static int32_t ocs_fw_dump_ready_wait(ocs_t *ocs, uint8_t dump_state)
{
    int32_t rc = 0;
    int32_t ms_waited = 0;
    int32_t wait_time = SLI4_FW_DUMP_TIMEOUT_MSEC;

    ocs_log_info(ocs, "Waiting for FW dump to be completed\n");
    while (!ocs_fw_dump_state_check(ocs, dump_state) && (ms_waited < wait_time)) {
        ocs_delay_usec(10000);
        ms_waited += 10;
    }

    if (!ocs_fw_dump_state_check(ocs, dump_state)) {
        ocs_log_err(ocs, "FW dump timedout=%d(msec)\n", ms_waited);
        rc = -1;
        goto exit_err;
    }

    ms_waited = 0;
    wait_time = SLI4_FW_READY_TIMEOUT_MSEC;
    /* Wait for recovery process is completed */
    if (ocs_recovery_state_check(OCS_RECOVERY_STATE_IN_PROGRESS)) {
        while (!ocs_recovery_state_check(OCS_RECOVERY_STATE_IDLE) && (ms_waited < wait_time)) {
            ocs_delay_usec(10000);
            ms_waited += 10;
        }

        if (!ocs_recovery_state_check(OCS_RECOVERY_STATE_IDLE)) {
            ocs_log_err(ocs, "Recovery timedout=%d(msec)\n", ms_waited);
            rc = -1;
            goto exit_err;
        }
    }

exit_err:
    return rc;
}

/**
 * @brief Copy dump buffers to user space buffer
 */
static inline int32_t ocs_fw_dump_copy_to_user(ocs_t *ocs, uint8_t *user_buff,
                uint32_t user_buflen, uint8_t dump_state)
{
    uint8_t *saved_buff = ocs->fw_dump.saved_buff;
    uint32_t size = ocs->fw_dump.size;

    if (!ocs_fw_dump_state_check(ocs, dump_state)) {
        ocs_log_warn(ocs, "FW dump is not present in the OCS buffers\n");
        return -1;
    }

    /* Copy the dump from the saved buffer to the user buffer */
    ocs_assert(user_buflen >= size, -1);
    if (ocs_copy_to_user(user_buff, saved_buff, size)) {
        ocs_log_err(ocs, "ocs_copy_to_user() failed\n");
        return -1;
    }

    return 0;
}

/**
 * @brief Create a Lancer dump into a memory buffer
 * @par Description
 * This function creates a DMA buffer to hold a Lancer dump,
 * sets the dump location to point to that buffer, then calls
 * ocs_gen_dump to cause a dump to be transfered to the buffer.
 * After the dump is complete it copies the dump to the provided
 * user space buffer.
 *
 * @param ocs Pointer to ocs structure
 * @param buf User space buffer in which to store the dump
 * @param buflen Length of the user buffer in bytes
 * @param dump_level Gendump request level can be either chip-level or fdb
 *
 * @return Returns 0 on success, non-zero on error.
 */
int32_t ocs_dump_to_host(ocs_t *ocs, void *buf, uint32_t buflen, uint8_t dump_level)
{
    ocs_t *other_ocs;
    ocs_dma_t *dump_buffers = NULL;
    uint8_t bus, dev, func;
    uint8_t other_bus, other_dev, other_func;
    int32_t rc = 0;
    uint32_t index = 0;
    uint32_t num_buffers = 0;

    if (FW_DUMP_TYPE_DUMPTOHOST != ocs->fw_dump.type) {
        ocs_log_err(ocs, "Dump to host is disabled\n");
        return -1;
    }

    if (!buflen) {
        ocs_log_err(ocs, "Zero buffer length is invalid\n");
        return -1;
    }

    if (FW_FUNC_DESC_DUMP == dump_level) {
        dump_buffers = ocs->fw_dump.func_dump_buffers;
        num_buffers = ocs->fw_dump.num_func_dump_buffers;
    } else {
        ocs_get_bus_dev_func(ocs, &bus, &dev, &func);
        while ((other_ocs = ocs_get_instance(index++)) != NULL) {
            ocs_get_bus_dev_func(other_ocs, &other_bus, &other_dev, &other_func);
            if ((bus == other_bus) && (dev == other_dev) && (0 == other_func)) {
                dump_buffers = other_ocs->fw_dump.chip_dump_buffers;
                num_buffers = other_ocs->fw_dump.num_chip_dump_buffers;
                break;
            }
        }
    }

    if (!dump_buffers || !num_buffers) {
        ocs_log_err(ocs, "No pre-allocated FW dump memory\n");
        return -1;
    }

    rc = ocs_initiate_gendump(ocs, dump_level);
    if (OCS_FW_DUMP_STATUS_FAILED == rc) {
        ocs_log_err(ocs, "ocs_initiate_gendump() failed\n");
        goto done;
    }

    if (OCS_FW_DUMP_STATUS_IN_PROGRESS == rc) {
        ocs_log_debug(ocs, "Waiting for the already triggered FW dump to complete\n");
        rc = ocs_fw_dump_ready_wait(ocs, dump_level);
    }

    if (OCS_FW_DUMP_STATUS_SUCCESS == rc) {
        /* Copy the dump from the DMA buffer into the user buffer */
        rc = ocs_fw_dump_copy_to_user(ocs, (uint8_t *)buf, buflen, dump_level);
        if (rc) {
            ocs_log_err(ocs, "FW dump copy to user failed\n");
        } else {
            ocs_log_info(ocs, "FW dump to host is successful\n");
        }
    }

done:
    ocs_fw_dump_state_set(ocs, FW_DUMP_STATE_NONE);
    return rc;
}

/**
 * @brief Recover adapter from chip error.
 */
static int32_t ocs_adapter_recover(ocs_t *ocs, uint32_t mode)
{
    uint8_t bus, dev, func;
    int32_t rc = 0;
    uint32_t err1 = sli_reg_read(&ocs->hal.sli, SLI4_REG_SLIPORT_ERROR1);
    uint32_t err2 = sli_reg_read(&ocs->hal.sli, SLI4_REG_SLIPORT_ERROR2);
    uint8_t dump_level = FW_CHIP_LEVEL_DUMP;
    bool trigger_dump = true;


    if (ocs_recovery_state_check(OCS_RECOVERY_STATE_IN_PROGRESS)) {
        ocs_log_info(ocs, "Adapter recovery is already in progress\n");
        return rc;
    }

    ocs_get_bus_dev_func(ocs, &bus, &dev, &func);
    ocs_log_crit(ocs, "OCS adapter %02d:%02d recovery started for UE "
            "status=%#x error1=%#x error2=%#x\n", bus, dev,
            sli_reg_read(&ocs->hal.sli, SLI4_REG_SLIPORT_STATUS),
            err1, err2);

    switch (err1) {
        /* Add the err1 codes here that impact only a single function */
        case SLI4_SLIPORT_ERROR1_RQ_EMPTY_TIMEOUT:
            dump_level = FW_FUNC_DESC_DUMP;
            trigger_dump = false;
            /* fall-through */
        default:
            break;
    }
    /* gendump and reset adapter */
    rc = ocs_recovery_reset(ocs, mode, dump_level, trigger_dump);
    if (OCS_RECOVERY_STATUS_IN_PROGRESS == rc) {
        rc = OCS_RECOVERY_STATUS_SUCCESS;
    }

    return rc;
}

/**
 * @ingroup os
 * @brief OCS FW error recovery thread
 *
 * @return 0 if success; < 0 otherwise.
 */
static int32_t ocs_recovery_thread(ocs_thread_t *th)
{
    ocs_t *ocs;
    int32_t rc, index;
    uint32_t fw_err_detected;

    while (!ocs_thread_terminate_requested(th)) {
        ocs_msleep(1000);

        if (ocs_recovery_state_check(OCS_RECOVERY_STATE_IN_PROGRESS))
            continue;

        index = 0;
        fw_err_detected = FALSE;
        while ((ocs = ocs_get_instance(index++)) != NULL) {
            if (sli_fw_error_status(&ocs->hal.sli) > 0) {
                fw_err_detected = TRUE;
                break;
            } else {
                fw_err_detected = FALSE;
            }
        }

        if (!fw_err_detected)
            continue;

        rc = ocs_adapter_recover(ocs, OCS_RECOVERY_MODE_AUTO);
        if (rc == 0) {
            if (ocs_recovery_state_check(OCS_RECOVERY_STATE_IN_PROGRESS)) {
                ocs_log_info(ocs, "FW error recovery is already in progress\n");
            } else {
                ocs_log_info(ocs, "FW error recovery is successful\n");
            }
        } else {
            ocs_log_err(ocs, "FW error recovery has failed\n");
        }
    }

    ocs_recovery_state_set(OCS_RECOVERY_STATE_IDLE);
    return 0;
}

/**
 * @brief Create a FW error recovery thread
 * 
 * @return returns 0 for success, a negative error code value for failure.
 */
int32_t ocs_fw_err_recovery_setup(void)
{
    int32_t rc;

    ocs_lock_init(NULL, &ocs_recovery.lock, "fw_err_recovery_lock");
    ocs_recovery.state = OCS_RECOVERY_STATE_IDLE;

    rc = ocs_thread_create(NULL, &ocs_recovery.thread,
                ocs_recovery_thread, "ocs_recovery",
                NULL, OCS_THREAD_RUN);
    if (rc < 0) {
        ocs_log_err(NULL, "Failed to create FW err detection thread\n");
    }

    return rc;
}

/**
 * @brief Initialize fw dump context
 */
void ocs_fw_dump_init(ocs_t *ocs, int32_t fw_dump_type)
{
    /* Init fw dump lock */
    ocs_lock_init(ocs, &ocs->fw_dump.lock, "fw_dump_lock[%d]", ocs_instance(ocs));
    ocs->fw_dump.state = FW_DUMP_STATE_NONE;
    ocs->fw_dump.type = fw_dump_type;
    ocs->fw_dump.chip_dump_buffers = NULL;
    ocs->fw_dump.func_dump_buffers = NULL;
    ocs->fw_dump.num_chip_dump_buffers = 0;
    ocs->fw_dump.num_func_dump_buffers = 0;
}