diff options
Diffstat (limited to 'drivers/scsi/lpfc/lpfc_sli.c')
| -rw-r--r-- | drivers/scsi/lpfc/lpfc_sli.c | 903 |
1 files changed, 721 insertions, 182 deletions
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index f9e6a135d656..64002b0cb02d 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -87,6 +87,10 @@ static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe); static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba); static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba); +static struct lpfc_cqe *lpfc_sli4_cq_get(struct lpfc_queue *q); +static void __lpfc_sli4_consume_cqe(struct lpfc_hba *phba, + struct lpfc_queue *cq, + struct lpfc_cqe *cqe); static IOCB_t * lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq) @@ -467,25 +471,52 @@ __lpfc_sli4_consume_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq, } static void -lpfc_sli4_eq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq) +lpfc_sli4_eqcq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq) { - struct lpfc_eqe *eqe; - uint32_t count = 0; + struct lpfc_eqe *eqe = NULL; + u32 eq_count = 0, cq_count = 0; + struct lpfc_cqe *cqe = NULL; + struct lpfc_queue *cq = NULL, *childq = NULL; + int cqid = 0; /* walk all the EQ entries and drop on the floor */ eqe = lpfc_sli4_eq_get(eq); while (eqe) { + /* Get the reference to the corresponding CQ */ + cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); + cq = NULL; + + list_for_each_entry(childq, &eq->child_list, list) { + if (childq->queue_id == cqid) { + cq = childq; + break; + } + } + /* If CQ is valid, iterate through it and drop all the CQEs */ + if (cq) { + cqe = lpfc_sli4_cq_get(cq); + while (cqe) { + __lpfc_sli4_consume_cqe(phba, cq, cqe); + cq_count++; + cqe = lpfc_sli4_cq_get(cq); + } + /* Clear and re-arm the CQ */ + phba->sli4_hba.sli4_write_cq_db(phba, cq, cq_count, + LPFC_QUEUE_REARM); + cq_count = 0; + } __lpfc_sli4_consume_eqe(phba, eq, eqe); - count++; + eq_count++; eqe = lpfc_sli4_eq_get(eq); } /* Clear and re-arm the EQ */ - phba->sli4_hba.sli4_write_eq_db(phba, eq, count, LPFC_QUEUE_REARM); + phba->sli4_hba.sli4_write_eq_db(phba, eq, eq_count, LPFC_QUEUE_REARM); } static int -lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq) +lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq, + uint8_t rearm) { struct lpfc_eqe *eqe; int count = 0, consumed = 0; @@ -519,8 +550,8 @@ lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq) eq->queue_claimed = 0; rearm_and_exit: - /* Always clear and re-arm the EQ */ - phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, LPFC_QUEUE_REARM); + /* Always clear the EQ. */ + phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, rearm); return count; } @@ -1391,9 +1422,12 @@ lpfc_sli_cancel_iocbs(struct lpfc_hba *phba, struct list_head *iocblist, while (!list_empty(iocblist)) { list_remove_head(iocblist, piocb, struct lpfc_iocbq, list); - if (!piocb->iocb_cmpl) - lpfc_sli_release_iocbq(phba, piocb); - else { + if (!piocb->iocb_cmpl) { + if (piocb->iocb_flag & LPFC_IO_NVME) + lpfc_nvme_cancel_iocb(phba, piocb); + else + lpfc_sli_release_iocbq(phba, piocb); + } else { piocb->iocb.ulpStatus = ulpstatus; piocb->iocb.un.ulpWord[4] = ulpWord4; (piocb->iocb_cmpl) (phba, piocb, piocb); @@ -2426,6 +2460,20 @@ lpfc_sli_wake_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) return; } +static void +__lpfc_sli_rpi_release(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) +{ + unsigned long iflags; + + if (ndlp->nlp_flag & NLP_RELEASE_RPI) { + lpfc_sli4_free_rpi(vport->phba, ndlp->nlp_rpi); + spin_lock_irqsave(&vport->phba->ndlp_lock, iflags); + ndlp->nlp_flag &= ~NLP_RELEASE_RPI; + ndlp->nlp_rpi = LPFC_RPI_ALLOC_ERROR; + spin_unlock_irqrestore(&vport->phba->ndlp_lock, iflags); + } + ndlp->nlp_flag &= ~NLP_UNREG_INP; +} /** * lpfc_sli_def_mbox_cmpl - Default mailbox completion handler @@ -2497,7 +2545,7 @@ lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) vport, KERN_INFO, LOG_MBOX | LOG_DISCOVERY, "1438 UNREG cmpl deferred mbox x%x " - "on NPort x%x Data: x%x x%x %p\n", + "on NPort x%x Data: x%x x%x %px\n", ndlp->nlp_rpi, ndlp->nlp_DID, ndlp->nlp_flag, ndlp->nlp_defer_did, ndlp); @@ -2507,8 +2555,10 @@ lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) ndlp->nlp_defer_did = NLP_EVT_NOTHING_PENDING; lpfc_issue_els_plogi(vport, ndlp->nlp_DID, 0); } else { - ndlp->nlp_flag &= ~NLP_UNREG_INP; + __lpfc_sli_rpi_release(vport, ndlp); } + if (vport->load_flag & FC_UNLOADING) + lpfc_nlp_put(ndlp); pmb->ctx_ndlp = NULL; } } @@ -2555,7 +2605,7 @@ lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) vport, KERN_INFO, LOG_MBOX | LOG_SLI, "0010 UNREG_LOGIN vpi:%x " "rpi:%x DID:%x defer x%x flg x%x " - "map:%x %p\n", + "map:%x %px\n", vport->vpi, ndlp->nlp_rpi, ndlp->nlp_DID, ndlp->nlp_defer_did, ndlp->nlp_flag, @@ -2573,7 +2623,7 @@ lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) vport, KERN_INFO, LOG_DISCOVERY, "4111 UNREG cmpl deferred " "clr x%x on " - "NPort x%x Data: x%x %p\n", + "NPort x%x Data: x%x x%px\n", ndlp->nlp_rpi, ndlp->nlp_DID, ndlp->nlp_defer_did, ndlp); ndlp->nlp_flag &= ~NLP_UNREG_INP; @@ -2582,7 +2632,7 @@ lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) lpfc_issue_els_plogi( vport, ndlp->nlp_DID, 0); } else { - ndlp->nlp_flag &= ~NLP_UNREG_INP; + __lpfc_sli_rpi_release(vport, ndlp); } } } @@ -2655,7 +2705,8 @@ lpfc_sli_handle_mb_event(struct lpfc_hba *phba) lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, "(%d):0323 Unknown Mailbox command " "x%x (x%x/x%x) Cmpl\n", - pmb->vport ? pmb->vport->vpi : 0, + pmb->vport ? pmb->vport->vpi : + LPFC_VPORT_UNKNOWN, pmbox->mbxCommand, lpfc_sli_config_mbox_subsys_get(phba, pmb), @@ -2676,7 +2727,8 @@ lpfc_sli_handle_mb_event(struct lpfc_hba *phba) "(%d):0305 Mbox cmd cmpl " "error - RETRYing Data: x%x " "(x%x/x%x) x%x x%x x%x\n", - pmb->vport ? pmb->vport->vpi : 0, + pmb->vport ? pmb->vport->vpi : + LPFC_VPORT_UNKNOWN, pmbox->mbxCommand, lpfc_sli_config_mbox_subsys_get(phba, pmb), @@ -2684,7 +2736,8 @@ lpfc_sli_handle_mb_event(struct lpfc_hba *phba) pmb), pmbox->mbxStatus, pmbox->un.varWords[0], - pmb->vport->port_state); + pmb->vport ? pmb->vport->port_state : + LPFC_VPORT_UNKNOWN); pmbox->mbxStatus = 0; pmbox->mbxOwner = OWN_HOST; rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT); @@ -2695,7 +2748,7 @@ lpfc_sli_handle_mb_event(struct lpfc_hba *phba) /* Mailbox cmd <cmd> Cmpl <cmpl> */ lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI, - "(%d):0307 Mailbox cmd x%x (x%x/x%x) Cmpl x%p " + "(%d):0307 Mailbox cmd x%x (x%x/x%x) Cmpl %ps " "Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x " "x%x x%x x%x\n", pmb->vport ? pmb->vport->vpi : 0, @@ -3961,7 +4014,7 @@ lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba) /* Look on all the FCP Rings for the iotag */ if (phba->sli_rev >= LPFC_SLI_REV4) { for (i = 0; i < phba->cfg_hdw_queue; i++) { - pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; + pring = phba->sli4_hba.hdwq[i].io_wq->pring; lpfc_sli_abort_iocb_ring(phba, pring); } } else { @@ -3971,17 +4024,17 @@ lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba) } /** - * lpfc_sli_flush_fcp_rings - flush all iocbs in the fcp ring + * lpfc_sli_flush_io_rings - flush all iocbs in the IO ring * @phba: Pointer to HBA context object. * - * This function flushes all iocbs in the fcp ring and frees all the iocb + * This function flushes all iocbs in the IO ring and frees all the iocb * objects in txq and txcmplq. This function will not issue abort iocbs * for all the iocb commands in txcmplq, they will just be returned with * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI * slot has been permanently disabled. **/ void -lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) +lpfc_sli_flush_io_rings(struct lpfc_hba *phba) { LIST_HEAD(txq); LIST_HEAD(txcmplq); @@ -3992,13 +4045,13 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) spin_lock_irq(&phba->hbalock); /* Indicate the I/O queues are flushed */ - phba->hba_flag |= HBA_FCP_IOQ_FLUSH; + phba->hba_flag |= HBA_IOQ_FLUSH; spin_unlock_irq(&phba->hbalock); /* Look on all the FCP Rings for the iotag */ if (phba->sli_rev >= LPFC_SLI_REV4) { for (i = 0; i < phba->cfg_hdw_queue; i++) { - pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; + pring = phba->sli4_hba.hdwq[i].io_wq->pring; spin_lock_irq(&pring->ring_lock); /* Retrieve everything on txq */ @@ -4046,56 +4099,6 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) } /** - * lpfc_sli_flush_nvme_rings - flush all wqes in the nvme rings - * @phba: Pointer to HBA context object. - * - * This function flushes all wqes in the nvme rings and frees all resources - * in the txcmplq. This function does not issue abort wqes for the IO - * commands in txcmplq, they will just be returned with - * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI - * slot has been permanently disabled. - **/ -void -lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba) -{ - LIST_HEAD(txcmplq); - struct lpfc_sli_ring *pring; - uint32_t i; - struct lpfc_iocbq *piocb, *next_iocb; - - if ((phba->sli_rev < LPFC_SLI_REV4) || - !(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) - return; - - /* Hint to other driver operations that a flush is in progress. */ - spin_lock_irq(&phba->hbalock); - phba->hba_flag |= HBA_NVME_IOQ_FLUSH; - spin_unlock_irq(&phba->hbalock); - - /* Cycle through all NVME rings and complete each IO with - * a local driver reason code. This is a flush so no - * abort exchange to FW. - */ - for (i = 0; i < phba->cfg_hdw_queue; i++) { - pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; - - spin_lock_irq(&pring->ring_lock); - list_for_each_entry_safe(piocb, next_iocb, - &pring->txcmplq, list) - piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; - /* Retrieve everything on the txcmplq */ - list_splice_init(&pring->txcmplq, &txcmplq); - pring->txcmplq_cnt = 0; - spin_unlock_irq(&pring->ring_lock); - - /* Flush the txcmpq &&&PAE */ - lpfc_sli_cancel_iocbs(phba, &txcmplq, - IOSTAT_LOCAL_REJECT, - IOERR_SLI_DOWN); - } -} - -/** * lpfc_sli_brdready_s3 - Check for sli3 host ready status * @phba: Pointer to HBA context object. * @mask: Bit mask to be checked. @@ -4495,7 +4498,7 @@ lpfc_sli_brdreset(struct lpfc_hba *phba) * checking during resets the device. The caller is not required to hold * any locks. * - * This function returns 0 always. + * This function returns 0 on success else returns negative error code. **/ int lpfc_sli4_brdreset(struct lpfc_hba *phba) @@ -4652,8 +4655,10 @@ lpfc_sli_brdrestart_s4(struct lpfc_hba *phba) hba_aer_enabled = phba->hba_flag & HBA_AER_ENABLED; rc = lpfc_sli4_brdreset(phba); - if (rc) - return rc; + if (rc) { + phba->link_state = LPFC_HBA_ERROR; + goto hba_down_queue; + } spin_lock_irq(&phba->hbalock); phba->pport->stopped = 0; @@ -4668,6 +4673,7 @@ lpfc_sli_brdrestart_s4(struct lpfc_hba *phba) if (hba_aer_enabled) pci_disable_pcie_error_reporting(phba->pcidev); +hba_down_queue: lpfc_hba_down_post(phba); lpfc_sli4_queue_destroy(phba); @@ -4912,8 +4918,17 @@ static int lpfc_sli4_rb_setup(struct lpfc_hba *phba) { phba->hbq_in_use = 1; - phba->hbqs[LPFC_ELS_HBQ].entry_count = - lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count; + /** + * Specific case when the MDS diagnostics is enabled and supported. + * The receive buffer count is truncated to manage the incoming + * traffic. + **/ + if (phba->cfg_enable_mds_diags && phba->mds_diags_support) + phba->hbqs[LPFC_ELS_HBQ].entry_count = + lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count >> 1; + else + phba->hbqs[LPFC_ELS_HBQ].entry_count = + lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count; phba->hbq_count = 1; lpfc_sli_hbqbuf_init_hbqs(phba, LPFC_ELS_HBQ); /* Initially populate or replenish the HBQs */ @@ -5584,10 +5599,8 @@ lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba) for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { qp = &sli4_hba->hdwq[qidx]; /* ARM the corresponding CQ */ - sli4_hba->sli4_write_cq_db(phba, qp->fcp_cq, 0, - LPFC_QUEUE_REARM); - sli4_hba->sli4_write_cq_db(phba, qp->nvme_cq, 0, - LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, qp->io_cq, 0, + LPFC_QUEUE_REARM); } /* Loop thru all IRQ vectors */ @@ -6199,6 +6212,14 @@ lpfc_set_features(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox, mbox->u.mqe.un.set_feature.feature = LPFC_SET_MDS_DIAGS; mbox->u.mqe.un.set_feature.param_len = 8; break; + case LPFC_SET_DUAL_DUMP: + bf_set(lpfc_mbx_set_feature_dd, + &mbox->u.mqe.un.set_feature, LPFC_ENABLE_DUAL_DUMP); + bf_set(lpfc_mbx_set_feature_ddquery, + &mbox->u.mqe.un.set_feature, 0); + mbox->u.mqe.un.set_feature.feature = LPFC_SET_DUAL_DUMP; + mbox->u.mqe.un.set_feature.param_len = 4; + break; } return; @@ -6216,11 +6237,16 @@ lpfc_ras_stop_fwlog(struct lpfc_hba *phba) { struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog; - ras_fwlog->ras_active = false; + spin_lock_irq(&phba->hbalock); + ras_fwlog->state = INACTIVE; + spin_unlock_irq(&phba->hbalock); /* Disable FW logging to host memory */ writel(LPFC_CTL_PDEV_CTL_DDL_RAS, phba->sli4_hba.conf_regs_memmap_p + LPFC_CTL_PDEV_CTL_OFFSET); + + /* Wait 10ms for firmware to stop using DMA buffer */ + usleep_range(10 * 1000, 20 * 1000); } /** @@ -6256,7 +6282,9 @@ lpfc_sli4_ras_dma_free(struct lpfc_hba *phba) ras_fwlog->lwpd.virt = NULL; } - ras_fwlog->ras_active = false; + spin_lock_irq(&phba->hbalock); + ras_fwlog->state = INACTIVE; + spin_unlock_irq(&phba->hbalock); } /** @@ -6358,7 +6386,9 @@ lpfc_sli4_ras_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) goto disable_ras; } - ras_fwlog->ras_active = true; + spin_lock_irq(&phba->hbalock); + ras_fwlog->state = ACTIVE; + spin_unlock_irq(&phba->hbalock); mempool_free(pmb, phba->mbox_mem_pool); return; @@ -6390,6 +6420,10 @@ lpfc_sli4_ras_fwlog_init(struct lpfc_hba *phba, uint32_t len = 0, fwlog_buffsize, fwlog_entry_count; int rc = 0; + spin_lock_irq(&phba->hbalock); + ras_fwlog->state = INACTIVE; + spin_unlock_irq(&phba->hbalock); + fwlog_buffsize = (LPFC_RAS_MIN_BUFF_POST_SIZE * phba->cfg_ras_fwlog_buffsize); fwlog_entry_count = (fwlog_buffsize/LPFC_RAS_MAX_ENTRY_SIZE); @@ -6449,6 +6483,9 @@ lpfc_sli4_ras_fwlog_init(struct lpfc_hba *phba, mbx_fwlog->u.request.lwpd.addr_lo = putPaddrLow(ras_fwlog->lwpd.phys); mbx_fwlog->u.request.lwpd.addr_hi = putPaddrHigh(ras_fwlog->lwpd.phys); + spin_lock_irq(&phba->hbalock); + ras_fwlog->state = REG_INPROGRESS; + spin_unlock_irq(&phba->hbalock); mbox->vport = phba->pport; mbox->mbox_cmpl = lpfc_sli4_ras_mbox_cmpl; @@ -7180,7 +7217,7 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, int lpfc_sli4_hba_setup(struct lpfc_hba *phba) { - int rc, i, cnt, len; + int rc, i, cnt, len, dd; LPFC_MBOXQ_t *mboxq; struct lpfc_mqe *mqe; uint8_t *vpd; @@ -7243,7 +7280,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) else phba->hba_flag &= ~HBA_FIP_SUPPORT; - phba->hba_flag &= ~HBA_FCP_IOQ_FLUSH; + phba->hba_flag &= ~HBA_IOQ_FLUSH; if (phba->sli_rev != LPFC_SLI_REV4) { lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, @@ -7431,6 +7468,23 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) phba->sli3_options |= (LPFC_SLI3_NPIV_ENABLED | LPFC_SLI3_HBQ_ENABLED); spin_unlock_irq(&phba->hbalock); + /* Always try to enable dual dump feature if we can */ + lpfc_set_features(phba, mboxq, LPFC_SET_DUAL_DUMP); + rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); + dd = bf_get(lpfc_mbx_set_feature_dd, &mboxq->u.mqe.un.set_feature); + if ((rc == MBX_SUCCESS) && (dd == LPFC_ENABLE_DUAL_DUMP)) + lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_INIT, + "6448 Dual Dump is enabled\n"); + else + lpfc_printf_log(phba, KERN_INFO, LOG_SLI | LOG_INIT, + "6447 Dual Dump Mailbox x%x (x%x/x%x) failed, " + "rc:x%x dd:x%x\n", + bf_get(lpfc_mqe_command, &mboxq->u.mqe), + lpfc_sli_config_mbox_subsys_get( + phba, mboxq), + lpfc_sli_config_mbox_opcode_get( + phba, mboxq), + rc, dd); /* * Allocate all resources (xri,rpi,vpi,vfi) now. Subsequent * calls depends on these resources to complete port setup. @@ -7555,9 +7609,11 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) } phba->sli4_hba.nvmet_xri_cnt = rc; - cnt = phba->cfg_iocb_cnt * 1024; - /* We need 1 iocbq for every SGL, for IO processing */ - cnt += phba->sli4_hba.nvmet_xri_cnt; + /* We allocate an iocbq for every receive context SGL. + * The additional allocation is for abort and ls handling. + */ + cnt = phba->sli4_hba.nvmet_xri_cnt + + phba->sli4_hba.max_cfg_param.max_xri; } else { /* update host common xri-sgl sizes and mappings */ rc = lpfc_sli4_io_sgl_update(phba); @@ -7579,14 +7635,17 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) rc = -ENODEV; goto out_destroy_queue; } - cnt = phba->cfg_iocb_cnt * 1024; + /* Each lpfc_io_buf job structure has an iocbq element. + * This cnt provides for abort, els, ct and ls requests. + */ + cnt = phba->sli4_hba.max_cfg_param.max_xri; } if (!phba->sli.iocbq_lookup) { /* Initialize and populate the iocb list per host */ lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "2821 initialize iocb list %d total %d\n", - phba->cfg_iocb_cnt, cnt); + "2821 initialize iocb list with %d entries\n", + cnt); rc = lpfc_init_iocb_list(phba, cnt); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -7898,7 +7957,7 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) if (sli4_hba->hdwq) { for (eqidx = 0; eqidx < phba->cfg_irq_chann; eqidx++) { eq = phba->sli4_hba.hba_eq_hdl[eqidx].eq; - if (eq->queue_id == sli4_hba->mbx_cq->assoc_qid) { + if (eq && eq->queue_id == sli4_hba->mbx_cq->assoc_qid) { fpeq = eq; break; } @@ -7924,7 +7983,7 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) if (mbox_pending) /* process and rearm the EQ */ - lpfc_sli4_process_eq(phba, fpeq); + lpfc_sli4_process_eq(phba, fpeq, LPFC_QUEUE_REARM); else /* Always clear and re-arm the EQ */ sli4_hba->sli4_write_eq_db(phba, fpeq, 0, LPFC_QUEUE_REARM); @@ -7972,7 +8031,7 @@ lpfc_mbox_timeout_handler(struct lpfc_hba *phba) /* Mbox cmd <mbxCommand> timeout */ lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "0310 Mailbox command x%x timeout Data: x%x x%x x%p\n", + "0310 Mailbox command x%x timeout Data: x%x x%x x%px\n", mb->mbxCommand, phba->pport->port_state, phba->sli.sli_flag, @@ -8505,7 +8564,7 @@ lpfc_sli4_async_mbox_unblock(struct lpfc_hba *phba) psli->sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK; spin_unlock_irq(&phba->hbalock); - /* wake up worker thread to post asynchronlous mailbox command */ + /* wake up worker thread to post asynchronous mailbox command */ lpfc_worker_wake_up(phba); } @@ -8773,7 +8832,7 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq, return rc; } - /* Now, interrupt mode asynchrous mailbox command */ + /* Now, interrupt mode asynchronous mailbox command */ rc = lpfc_mbox_cmd_check(phba, mboxq); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, @@ -8996,7 +9055,8 @@ lpfc_mbox_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) * @pring: Pointer to driver SLI ring object. * @piocb: Pointer to address of newly added command iocb. * - * This function is called with hbalock held to add a command + * This function is called with hbalock held for SLI3 ports or + * the ring lock held for SLI4 ports to add a command * iocb to the txq when SLI layer cannot submit the command iocb * to the ring. **/ @@ -9004,7 +9064,10 @@ void __lpfc_sli_ringtx_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, struct lpfc_iocbq *piocb) { - lockdep_assert_held(&phba->hbalock); + if (phba->sli_rev == LPFC_SLI_REV4) + lockdep_assert_held(&pring->ring_lock); + else + lockdep_assert_held(&phba->hbalock); /* Insert the caller's iocb in the txq tail for later processing. */ list_add_tail(&piocb->list, &pring->txq); } @@ -9333,11 +9396,9 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, memset(wqe, 0, sizeof(union lpfc_wqe128)); /* Some of the fields are in the right position already */ memcpy(wqe, &iocbq->iocb, sizeof(union lpfc_wqe)); - if (iocbq->iocb.ulpCommand != CMD_SEND_FRAME) { - /* The ct field has moved so reset */ - wqe->generic.wqe_com.word7 = 0; - wqe->generic.wqe_com.word10 = 0; - } + /* The ct field has moved so reset */ + wqe->generic.wqe_com.word7 = 0; + wqe->generic.wqe_com.word10 = 0; abort_tag = (uint32_t) iocbq->iotag; xritag = iocbq->sli4_xritag; @@ -9796,7 +9857,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, * we re-construct this WQE here based on information in * iocbq from scratch. */ - memset(wqe, 0, sizeof(union lpfc_wqe)); + memset(wqe, 0, sizeof(*wqe)); /* OX_ID is invariable to who sent ABTS to CT exchange */ bf_set(xmit_bls_rsp64_oxid, &wqe->xmit_bls_rsp, bf_get(lpfc_abts_oxid, &iocbq->iocb.un.bls_rsp)); @@ -9843,6 +9904,15 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, break; case CMD_SEND_FRAME: + bf_set(wqe_cmnd, &wqe->generic.wqe_com, CMD_SEND_FRAME); + bf_set(wqe_sof, &wqe->generic.wqe_com, 0x2E); /* SOF byte */ + bf_set(wqe_eof, &wqe->generic.wqe_com, 0x41); /* EOF byte */ + bf_set(wqe_lenloc, &wqe->generic.wqe_com, 1); + bf_set(wqe_xbl, &wqe->generic.wqe_com, 1); + bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); + bf_set(wqe_xc, &wqe->generic.wqe_com, 1); + bf_set(wqe_cmd_type, &wqe->generic.wqe_com, 0xA); + bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); bf_set(wqe_xri_tag, &wqe->generic.wqe_com, xritag); bf_set(wqe_reqtag, &wqe->generic.wqe_com, iocbq->iotag); return 0; @@ -9888,7 +9958,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, * __lpfc_sli_issue_iocb_s4 is used by other functions in the driver to issue * an iocb command to an HBA with SLI-4 interface spec. * - * This function is called with hbalock held. The function will return success + * This function is called with ringlock held. The function will return success * after it successfully submit the iocb to firmware or after adding to the * txq. **/ @@ -9904,7 +9974,7 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number, /* Get the WQ */ if ((piocb->iocb_flag & LPFC_IO_FCP) || (piocb->iocb_flag & LPFC_USE_FCPWQIDX)) { - wq = phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq; + wq = phba->sli4_hba.hdwq[piocb->hba_wqidx].io_wq; } else { wq = phba->sli4_hba.els_wq; } @@ -10051,7 +10121,7 @@ lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) lpfc_cmd = (struct lpfc_io_buf *)piocb->context1; piocb->hba_wqidx = lpfc_cmd->hdwq_no; } - return phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq->pring; + return phba->sli4_hba.hdwq[piocb->hba_wqidx].io_wq->pring; } else { if (unlikely(!phba->sli4_hba.els_wq)) return NULL; @@ -10078,10 +10148,13 @@ lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, struct lpfc_iocbq *piocb, uint32_t flag) { struct lpfc_sli_ring *pring; + struct lpfc_queue *eq; unsigned long iflags; int rc; if (phba->sli_rev == LPFC_SLI_REV4) { + eq = phba->sli4_hba.hdwq[piocb->hba_wqidx].hba_eq; + pring = lpfc_sli4_calc_ring(phba, piocb); if (unlikely(pring == NULL)) return IOCB_ERROR; @@ -10089,6 +10162,8 @@ lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, spin_lock_irqsave(&pring->ring_lock, iflags); rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag); spin_unlock_irqrestore(&pring->ring_lock, iflags); + + lpfc_sli4_poll_eq(eq, LPFC_POLL_FASTPATH); } else { /* For now, SLI2/3 will still use hbalock */ spin_lock_irqsave(&phba->hbalock, iflags); @@ -10504,7 +10579,7 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) INIT_LIST_HEAD(&psli->mboxq_cmpl); /* Initialize list headers for txq and txcmplq as double linked lists */ for (i = 0; i < phba->cfg_hdw_queue; i++) { - pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; + pring = phba->sli4_hba.hdwq[i].io_wq->pring; pring->flag = 0; pring->ringno = LPFC_FCP_RING; pring->txcmplq_cnt = 0; @@ -10523,16 +10598,6 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) spin_lock_init(&pring->ring_lock); if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - for (i = 0; i < phba->cfg_hdw_queue; i++) { - pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; - pring->flag = 0; - pring->ringno = LPFC_FCP_RING; - pring->txcmplq_cnt = 0; - INIT_LIST_HEAD(&pring->txq); - INIT_LIST_HEAD(&pring->txcmplq); - INIT_LIST_HEAD(&pring->iocb_continueq); - spin_lock_init(&pring->ring_lock); - } pring = phba->sli4_hba.nvmels_wq->pring; pring->flag = 0; pring->ringno = LPFC_ELS_RING; @@ -10713,14 +10778,14 @@ lpfc_sli_host_down(struct lpfc_vport *vport) set_bit(LPFC_DATA_READY, &phba->data_flags); } prev_pring_flag = pring->flag; - spin_lock_irq(&pring->ring_lock); + spin_lock(&pring->ring_lock); list_for_each_entry_safe(iocb, next_iocb, &pring->txq, list) { if (iocb->vport != vport) continue; list_move_tail(&iocb->list, &completions); } - spin_unlock_irq(&pring->ring_lock); + spin_unlock(&pring->ring_lock); list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list) { if (iocb->vport != vport) @@ -10796,9 +10861,9 @@ lpfc_sli_hba_down(struct lpfc_hba *phba) pring = qp->pring; if (!pring) continue; - spin_lock_irq(&pring->ring_lock); + spin_lock(&pring->ring_lock); list_splice_init(&pring->txq, &completions); - spin_unlock_irq(&pring->ring_lock); + spin_unlock(&pring->ring_lock); if (pring == phba->sli4_hba.els_wq->pring) { pring->flag |= LPFC_DEFERRED_RING_EVENT; /* Set the lpfc data pending flag */ @@ -10979,7 +11044,7 @@ lpfc_sli_ring_taggedbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, spin_unlock_irq(&phba->hbalock); lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0402 Cannot find virtual addr for buffer tag on " - "ring %d Data x%lx x%p x%p x%x\n", + "ring %d Data x%lx x%px x%px x%x\n", pring->ringno, (unsigned long) tag, slp->next, slp->prev, pring->postbufq_cnt); @@ -11023,7 +11088,7 @@ lpfc_sli_ringpostbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, spin_unlock_irq(&phba->hbalock); lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0410 Cannot find virtual addr for mapped buf on " - "ring %d Data x%llx x%p x%p x%x\n", + "ring %d Data x%llx x%px x%px x%x\n", pring->ringno, (unsigned long long)phys, slp->next, slp->prev, pring->postbufq_cnt); return NULL; @@ -11078,7 +11143,7 @@ lpfc_sli_abort_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, abort_iocb = phba->sli.iocbq_lookup[abort_context]; lpfc_printf_log(phba, KERN_WARNING, LOG_ELS | LOG_SLI, - "0327 Cannot abort els iocb %p " + "0327 Cannot abort els iocb x%px " "with tag %x context %x, abort status %x, " "abort code %x\n", abort_iocb, abort_iotag, abort_context, @@ -11493,7 +11558,7 @@ lpfc_sli_abort_iocb(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, int i; /* all I/Os are in process of being flushed */ - if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) + if (phba->hba_flag & HBA_IOQ_FLUSH) return errcnt; for (i = 1; i <= phba->sli.last_iotag; i++) { @@ -11603,7 +11668,7 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, spin_lock_irqsave(&phba->hbalock, iflags); /* all I/Os are in process of being flushed */ - if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) { + if (phba->hba_flag & HBA_IOQ_FLUSH) { spin_unlock_irqrestore(&phba->hbalock, iflags); return 0; } @@ -11627,7 +11692,7 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, if (phba->sli_rev == LPFC_SLI_REV4) { pring_s4 = - phba->sli4_hba.hdwq[iocbq->hba_wqidx].fcp_wq->pring; + phba->sli4_hba.hdwq[iocbq->hba_wqidx].io_wq->pring; if (!pring_s4) { spin_unlock(&lpfc_cmd->buf_lock); continue; @@ -11768,7 +11833,10 @@ lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba, !(cmdiocbq->iocb_flag & LPFC_IO_LIBDFC)) { lpfc_cmd = container_of(cmdiocbq, struct lpfc_io_buf, cur_iocbq); - lpfc_cmd->exch_busy = rspiocbq->iocb_flag & LPFC_EXCHANGE_BUSY; + if (rspiocbq && (rspiocbq->iocb_flag & LPFC_EXCHANGE_BUSY)) + lpfc_cmd->flags |= LPFC_SBUF_XBUSY; + else + lpfc_cmd->flags &= ~LPFC_SBUF_XBUSY; } pdone_q = cmdiocbq->context_un.wait_queue; @@ -13053,11 +13121,11 @@ lpfc_cq_event_setup(struct lpfc_hba *phba, void *entry, int size) } /** - * lpfc_sli4_sp_handle_async_event - Handle an asynchroous event + * lpfc_sli4_sp_handle_async_event - Handle an asynchronous event * @phba: Pointer to HBA context object. * @cqe: Pointer to mailbox completion queue entry. * - * This routine process a mailbox completion queue entry with asynchrous + * This routine process a mailbox completion queue entry with asynchronous * event. * * Return: true if work posted to worker thread, otherwise false. @@ -13190,13 +13258,19 @@ send_current_mbox: phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE; /* Setting active mailbox pointer need to be in sync to flag clear */ phba->sli.mbox_active = NULL; + if (bf_get(lpfc_trailer_consumed, mcqe)) + lpfc_sli4_mq_release(phba->sli4_hba.mbx_wq); spin_unlock_irqrestore(&phba->hbalock, iflags); /* Wake up worker thread to post the next pending mailbox command */ lpfc_worker_wake_up(phba); + return workposted; + out_no_mqe_complete: + spin_lock_irqsave(&phba->hbalock, iflags); if (bf_get(lpfc_trailer_consumed, mcqe)) lpfc_sli4_mq_release(phba->sli4_hba.mbx_wq); - return workposted; + spin_unlock_irqrestore(&phba->hbalock, iflags); + return false; } /** @@ -13205,7 +13279,7 @@ out_no_mqe_complete: * @cqe: Pointer to mailbox completion queue entry. * * This routine process a mailbox completion queue entry, it invokes the - * proper mailbox complete handling or asynchrous event handling routine + * proper mailbox complete handling or asynchronous event handling routine * according to the MCQE's async bit. * * Return: true if work posted to worker thread, otherwise false. @@ -13249,7 +13323,6 @@ lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, struct lpfc_sli_ring *pring = cq->pring; int txq_cnt = 0; int txcmplq_cnt = 0; - int fcp_txcmplq_cnt = 0; /* Check for response status */ if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) { @@ -13271,9 +13344,8 @@ lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, txcmplq_cnt++; lpfc_printf_log(phba, KERN_ERR, LOG_SLI, "0387 NO IOCBQ data: txq_cnt=%d iocb_cnt=%d " - "fcp_txcmplq_cnt=%d, els_txcmplq_cnt=%d\n", + "els_txcmplq_cnt=%d\n", txq_cnt, phba->iocb_cnt, - fcp_txcmplq_cnt, txcmplq_cnt); return false; } @@ -13336,8 +13408,13 @@ lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba, unsigned long iflags; switch (cq->subtype) { - case LPFC_FCP: - lpfc_sli4_fcp_xri_aborted(phba, wcqe, cq->hdwq); + case LPFC_IO: + lpfc_sli4_io_xri_aborted(phba, wcqe, cq->hdwq); + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + /* Notify aborted XRI for NVME work queue */ + if (phba->nvmet_support) + lpfc_sli4_nvmet_xri_aborted(phba, wcqe); + } workposted = false; break; case LPFC_NVME_LS: /* NVME LS uses ELS resources */ @@ -13355,15 +13432,6 @@ lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba, spin_unlock_irqrestore(&phba->hbalock, iflags); workposted = true; break; - case LPFC_NVME: - /* Notify aborted XRI for NVME work queue */ - if (phba->nvmet_support) - lpfc_sli4_nvmet_xri_aborted(phba, wcqe); - else - lpfc_sli4_nvme_xri_aborted(phba, wcqe, cq->hdwq); - - workposted = false; - break; default: lpfc_printf_log(phba, KERN_ERR, LOG_SLI, "0603 Invalid CQ subtype %d: " @@ -13628,6 +13696,7 @@ __lpfc_sli4_process_cq(struct lpfc_hba *phba, struct lpfc_queue *cq, phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed, LPFC_QUEUE_NOARM); consumed = 0; + cq->assoc_qp->q_flag |= HBA_EQ_DELAY_CHK; } if (count == LPFC_NVMET_CQ_NOTIFY) @@ -13691,7 +13760,7 @@ __lpfc_sli4_sp_process_cq(struct lpfc_queue *cq) &delay); break; case LPFC_WCQ: - if (cq->subtype == LPFC_FCP || cq->subtype == LPFC_NVME) + if (cq->subtype == LPFC_IO) workposted |= __lpfc_sli4_process_cq(phba, cq, lpfc_sli4_fp_handle_cqe, &delay); @@ -14008,10 +14077,7 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, cq->CQ_wq++; /* Process the WQ complete event */ phba->last_completion_time = jiffies; - if ((cq->subtype == LPFC_FCP) || (cq->subtype == LPFC_NVME)) - lpfc_sli4_fp_handle_fcp_wcqe(phba, cq, - (struct lpfc_wcqe_complete *)&wcqe); - if (cq->subtype == LPFC_NVME_LS) + if (cq->subtype == LPFC_IO || cq->subtype == LPFC_NVME_LS) lpfc_sli4_fp_handle_fcp_wcqe(phba, cq, (struct lpfc_wcqe_complete *)&wcqe); break; @@ -14259,7 +14325,7 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) spin_lock_irqsave(&phba->hbalock, iflag); if (phba->link_state < LPFC_LINK_DOWN) /* Flush, clear interrupt, and rearm the EQ */ - lpfc_sli4_eq_flush(phba, fpeq); + lpfc_sli4_eqcq_flush(phba, fpeq); spin_unlock_irqrestore(&phba->hbalock, iflag); return IRQ_NONE; } @@ -14269,14 +14335,14 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) fpeq->last_cpu = raw_smp_processor_id(); if (icnt > LPFC_EQD_ISR_TRIGGER && - phba->cfg_irq_chann == 1 && + fpeq->q_flag & HBA_EQ_DELAY_CHK && phba->cfg_auto_imax && fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY && phba->sli.sli_flag & LPFC_SLI_USE_EQDR) lpfc_sli4_mod_hba_eq_delay(phba, fpeq, LPFC_MAX_AUTO_EQ_DELAY); /* process and rearm the EQ */ - ecount = lpfc_sli4_process_eq(phba, fpeq); + ecount = lpfc_sli4_process_eq(phba, fpeq, LPFC_QUEUE_REARM); if (unlikely(ecount == 0)) { fpeq->EQ_no_entry++; @@ -14336,6 +14402,147 @@ lpfc_sli4_intr_handler(int irq, void *dev_id) return (hba_handled == true) ? IRQ_HANDLED : IRQ_NONE; } /* lpfc_sli4_intr_handler */ +void lpfc_sli4_poll_hbtimer(struct timer_list *t) +{ + struct lpfc_hba *phba = from_timer(phba, t, cpuhp_poll_timer); + struct lpfc_queue *eq; + int i = 0; + + rcu_read_lock(); + + list_for_each_entry_rcu(eq, &phba->poll_list, _poll_list) + i += lpfc_sli4_poll_eq(eq, LPFC_POLL_SLOWPATH); + if (!list_empty(&phba->poll_list)) + mod_timer(&phba->cpuhp_poll_timer, + jiffies + msecs_to_jiffies(LPFC_POLL_HB)); + + rcu_read_unlock(); +} + +inline int lpfc_sli4_poll_eq(struct lpfc_queue *eq, uint8_t path) +{ + struct lpfc_hba *phba = eq->phba; + int i = 0; + + /* + * Unlocking an irq is one of the entry point to check + * for re-schedule, but we are good for io submission + * path as midlayer does a get_cpu to glue us in. Flush + * out the invalidate queue so we can see the updated + * value for flag. + */ + smp_rmb(); + + if (READ_ONCE(eq->mode) == LPFC_EQ_POLL) + /* We will not likely get the completion for the caller + * during this iteration but i guess that's fine. + * Future io's coming on this eq should be able to + * pick it up. As for the case of single io's, they + * will be handled through a sched from polling timer + * function which is currently triggered every 1msec. + */ + i = lpfc_sli4_process_eq(phba, eq, LPFC_QUEUE_NOARM); + + return i; +} + +static inline void lpfc_sli4_add_to_poll_list(struct lpfc_queue *eq) +{ + struct lpfc_hba *phba = eq->phba; + + if (list_empty(&phba->poll_list)) { + timer_setup(&phba->cpuhp_poll_timer, lpfc_sli4_poll_hbtimer, 0); + /* kickstart slowpath processing for this eq */ + mod_timer(&phba->cpuhp_poll_timer, + jiffies + msecs_to_jiffies(LPFC_POLL_HB)); + } + + list_add_rcu(&eq->_poll_list, &phba->poll_list); + synchronize_rcu(); +} + +static inline void lpfc_sli4_remove_from_poll_list(struct lpfc_queue *eq) +{ + struct lpfc_hba *phba = eq->phba; + + /* Disable slowpath processing for this eq. Kick start the eq + * by RE-ARMING the eq's ASAP + */ + list_del_rcu(&eq->_poll_list); + synchronize_rcu(); + + if (list_empty(&phba->poll_list)) + del_timer_sync(&phba->cpuhp_poll_timer); +} + +void lpfc_sli4_cleanup_poll_list(struct lpfc_hba *phba) +{ + struct lpfc_queue *eq, *next; + + list_for_each_entry_safe(eq, next, &phba->poll_list, _poll_list) + list_del(&eq->_poll_list); + + INIT_LIST_HEAD(&phba->poll_list); + synchronize_rcu(); +} + +static inline void +__lpfc_sli4_switch_eqmode(struct lpfc_queue *eq, uint8_t mode) +{ + if (mode == eq->mode) + return; + /* + * currently this function is only called during a hotplug + * event and the cpu on which this function is executing + * is going offline. By now the hotplug has instructed + * the scheduler to remove this cpu from cpu active mask. + * So we don't need to work about being put aside by the + * scheduler for a high priority process. Yes, the inte- + * rrupts could come but they are known to retire ASAP. + */ + + /* Disable polling in the fastpath */ + WRITE_ONCE(eq->mode, mode); + /* flush out the store buffer */ + smp_wmb(); + + /* + * Add this eq to the polling list and start polling. For + * a grace period both interrupt handler and poller will + * try to process the eq _but_ that's fine. We have a + * synchronization mechanism in place (queue_claimed) to + * deal with it. This is just a draining phase for int- + * errupt handler (not eq's) as we have guranteed through + * barrier that all the CPUs have seen the new CQ_POLLED + * state. which will effectively disable the REARMING of + * the EQ. The whole idea is eq's die off eventually as + * we are not rearming EQ's anymore. + */ + mode ? lpfc_sli4_add_to_poll_list(eq) : + lpfc_sli4_remove_from_poll_list(eq); +} + +void lpfc_sli4_start_polling(struct lpfc_queue *eq) +{ + __lpfc_sli4_switch_eqmode(eq, LPFC_EQ_POLL); +} + +void lpfc_sli4_stop_polling(struct lpfc_queue *eq) +{ + struct lpfc_hba *phba = eq->phba; + + __lpfc_sli4_switch_eqmode(eq, LPFC_EQ_INTERRUPT); + + /* Kick start for the pending io's in h/w. + * Once we switch back to interrupt processing on a eq + * the io path completion will only arm eq's when it + * receives a completion. But since eq's are in disa- + * rmed state it doesn't receive a completion. This + * creates a deadlock scenaro. + */ + phba->sli4_hba.sli4_write_eq_db(phba, eq, 0, LPFC_QUEUE_REARM); +} + /** * lpfc_sli4_queue_free - free a queue structure and associated memory * @queue: The queue structure to free. @@ -14410,6 +14617,7 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size, return NULL; INIT_LIST_HEAD(&queue->list); + INIT_LIST_HEAD(&queue->_poll_list); INIT_LIST_HEAD(&queue->wq_list); INIT_LIST_HEAD(&queue->wqfull_list); INIT_LIST_HEAD(&queue->page_list); @@ -16918,6 +17126,8 @@ lpfc_fc_frame_check(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr) struct fc_vft_header *fc_vft_hdr; uint32_t *header = (uint32_t *) fc_hdr; +#define FC_RCTL_MDS_DIAGS 0xF4 + switch (fc_hdr->fh_r_ctl) { case FC_RCTL_DD_UNCAT: /* uncategorized information */ case FC_RCTL_DD_SOL_DATA: /* solicited data */ @@ -17445,7 +17655,6 @@ lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport, icmd->ulpContext = phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]; ctiocb->context1 = lpfc_nlp_get(ndlp); - ctiocb->iocb_cmpl = NULL; ctiocb->vport = phba->pport; ctiocb->iocb_cmpl = lpfc_sli4_seq_abort_rsp_cmpl; ctiocb->sli4_lxritag = NO_XRI; @@ -17928,6 +18137,17 @@ lpfc_sli4_handle_received_buffer(struct lpfc_hba *phba, fcfi = bf_get(lpfc_rcqe_fcf_id, &dmabuf->cq_event.cqe.rcqe_cmpl); + if (fc_hdr->fh_r_ctl == 0xF4 && fc_hdr->fh_type == 0xFF) { + vport = phba->pport; + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "2023 MDS Loopback %d bytes\n", + bf_get(lpfc_rcqe_length, + &dmabuf->cq_event.cqe.rcqe_cmpl)); + /* Handle MDS Loopback frames */ + lpfc_sli4_handle_mds_loopback(vport, dmabuf); + return; + } + /* d_id this frame is directed to */ did = sli4_did_from_fc_hdr(fc_hdr); @@ -18151,8 +18371,9 @@ lpfc_sli4_alloc_rpi(struct lpfc_hba *phba) phba->sli4_hba.max_cfg_param.rpi_used++; phba->sli4_hba.rpi_count++; } - lpfc_printf_log(phba, KERN_INFO, LOG_SLI, - "0001 rpi:%x max:%x lim:%x\n", + lpfc_printf_log(phba, KERN_INFO, + LOG_NODE | LOG_DISCOVERY, + "0001 Allocated rpi:x%x max:x%x lim:x%x\n", (int) rpi, max_rpi, rpi_limit); /* @@ -18208,9 +18429,21 @@ lpfc_sli4_alloc_rpi(struct lpfc_hba *phba) static void __lpfc_sli4_free_rpi(struct lpfc_hba *phba, int rpi) { + /* + * if the rpi value indicates a prior unreg has already + * been done, skip the unreg. + */ + if (rpi == LPFC_RPI_ALLOC_ERROR) + return; + if (test_and_clear_bit(rpi, phba->sli4_hba.rpi_bmask)) { phba->sli4_hba.rpi_count--; phba->sli4_hba.max_cfg_param.rpi_used--; + } else { + lpfc_printf_log(phba, KERN_INFO, + LOG_NODE | LOG_DISCOVERY, + "2016 rpi %x not inuse\n", + rpi); } } @@ -19225,7 +19458,7 @@ lpfc_wr_object(struct lpfc_hba *phba, struct list_head *dmabuf_list, struct lpfc_mbx_wr_object *wr_object; LPFC_MBOXQ_t *mbox; int rc = 0, i = 0; - uint32_t shdr_status, shdr_add_status, shdr_change_status; + uint32_t shdr_status, shdr_add_status, shdr_change_status, shdr_csf; uint32_t mbox_tmo; struct lpfc_dmabuf *dmabuf; uint32_t written = 0; @@ -19282,6 +19515,16 @@ lpfc_wr_object(struct lpfc_hba *phba, struct list_head *dmabuf_list, if (check_change_status) { shdr_change_status = bf_get(lpfc_wr_object_change_status, &wr_object->u.response); + + if (shdr_change_status == LPFC_CHANGE_STATUS_FW_RESET || + shdr_change_status == LPFC_CHANGE_STATUS_PORT_MIGRATION) { + shdr_csf = bf_get(lpfc_wr_object_csf, + &wr_object->u.response); + if (shdr_csf) + shdr_change_status = + LPFC_CHANGE_STATUS_PCI_RESET; + } + switch (shdr_change_status) { case (LPFC_CHANGE_STATUS_PHYS_DEV_RESET): lpfc_printf_log(phba, KERN_INFO, LOG_INIT, @@ -19461,7 +19704,7 @@ lpfc_drain_txq(struct lpfc_hba *phba) if (phba->link_flag & LS_MDS_LOOPBACK) { /* MDS WQE are posted only to first WQ*/ - wq = phba->sli4_hba.hdwq[0].fcp_wq; + wq = phba->sli4_hba.hdwq[0].io_wq; if (unlikely(!wq)) return 0; pring = wq->pring; @@ -19706,16 +19949,18 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, lpfc_sli_ringtxcmpl_put(phba, pring, pwqe); spin_unlock_irqrestore(&pring->ring_lock, iflags); + + lpfc_sli4_poll_eq(qp->hba_eq, LPFC_POLL_FASTPATH); return 0; } /* NVME_FCREQ and NVME_ABTS requests */ if (pwqe->iocb_flag & LPFC_IO_NVME) { /* Get the IO distribution (hba_wqidx) for WQ assignment. */ - wq = qp->nvme_wq; + wq = qp->io_wq; pring = wq->pring; - bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->nvme_cq_map); + bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map); lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, qp, wq_access); @@ -19726,13 +19971,15 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, } lpfc_sli_ringtxcmpl_put(phba, pring, pwqe); spin_unlock_irqrestore(&pring->ring_lock, iflags); + + lpfc_sli4_poll_eq(qp->hba_eq, LPFC_POLL_FASTPATH); return 0; } /* NVMET requests */ if (pwqe->iocb_flag & LPFC_IO_NVMET) { /* Get the IO distribution (hba_wqidx) for WQ assignment. */ - wq = qp->nvme_wq; + wq = qp->io_wq; pring = wq->pring; ctxp = pwqe->context2; @@ -19743,7 +19990,7 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, } bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com, pwqe->sli4_xritag); - bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->nvme_cq_map); + bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map); lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, qp, wq_access); @@ -19754,6 +20001,8 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, } lpfc_sli_ringtxcmpl_put(phba, pring, pwqe); spin_unlock_irqrestore(&pring->ring_lock, iflags); + + lpfc_sli4_poll_eq(qp->hba_eq, LPFC_POLL_FASTPATH); return 0; } return WQE_ERROR; @@ -19790,9 +20039,7 @@ void lpfc_snapshot_mxp(struct lpfc_hba *phba, u32 hwqid) if (multixri_pool->stat_snapshot_taken == LPFC_MXP_SNAPSHOT_TAKEN) { pvt_pool = &qp->p_multixri_pool->pvt_pool; pbl_pool = &qp->p_multixri_pool->pbl_pool; - txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; - if (qp->nvme_wq) - txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; + txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; multixri_pool->stat_pbl_count = pbl_pool->count; multixri_pool->stat_pvt_count = pvt_pool->count; @@ -19862,12 +20109,9 @@ void lpfc_adjust_high_watermark(struct lpfc_hba *phba, u32 hwqid) watermark_max = xri_limit; watermark_min = xri_limit / 2; - txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; abts_io_bufs = qp->abts_scsi_io_bufs; - if (qp->nvme_wq) { - txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; - abts_io_bufs += qp->abts_nvme_io_bufs; - } + abts_io_bufs += qp->abts_nvme_io_bufs; new_watermark = txcmplq_cnt + abts_io_bufs; new_watermark = min(watermark_max, new_watermark); @@ -20121,6 +20365,13 @@ void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd, lpfc_ncmd->cur_iocbq.wqe_cmpl = NULL; lpfc_ncmd->cur_iocbq.iocb_cmpl = NULL; + if (phba->cfg_xpsgl && !phba->nvmet_support && + !list_empty(&lpfc_ncmd->dma_sgl_xtra_list)) + lpfc_put_sgl_per_hdwq(phba, lpfc_ncmd); + + if (!list_empty(&lpfc_ncmd->dma_cmd_rsp_list)) + lpfc_put_cmd_rsp_buf_per_hdwq(phba, lpfc_ncmd); + if (phba->cfg_xri_rebalancing) { if (lpfc_ncmd->expedite) { /* Return to expedite pool */ @@ -20142,12 +20393,9 @@ void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd, pbl_pool = &qp->p_multixri_pool->pbl_pool; pvt_pool = &qp->p_multixri_pool->pvt_pool; - txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; abts_io_bufs = qp->abts_scsi_io_bufs; - if (qp->nvme_wq) { - txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; - abts_io_bufs += qp->abts_nvme_io_bufs; - } + abts_io_bufs += qp->abts_nvme_io_bufs; xri_owned = pvt_pool->count + txcmplq_cnt + abts_io_bufs; xri_limit = qp->p_multixri_pool->xri_limit; @@ -20402,3 +20650,294 @@ struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba, return lpfc_cmd; } + +/** + * lpfc_get_sgl_per_hdwq - Get one SGL chunk from hdwq's pool + * @phba: The HBA for which this call is being executed. + * @lpfc_buf: IO buf structure to append the SGL chunk + * + * This routine gets one SGL chunk buffer from hdwq's SGL chunk pool, + * and will allocate an SGL chunk if the pool is empty. + * + * Return codes: + * NULL - Error + * Pointer to sli4_hybrid_sgl - Success + **/ +struct sli4_hybrid_sgl * +lpfc_get_sgl_per_hdwq(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_buf) +{ + struct sli4_hybrid_sgl *list_entry = NULL; + struct sli4_hybrid_sgl *tmp = NULL; + struct sli4_hybrid_sgl *allocated_sgl = NULL; + struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq; + struct list_head *buf_list = &hdwq->sgl_list; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + if (likely(!list_empty(buf_list))) { + /* break off 1 chunk from the sgl_list */ + list_for_each_entry_safe(list_entry, tmp, + buf_list, list_node) { + list_move_tail(&list_entry->list_node, + &lpfc_buf->dma_sgl_xtra_list); + break; + } + } else { + /* allocate more */ + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + tmp = kmalloc_node(sizeof(*tmp), GFP_ATOMIC, + cpu_to_node(hdwq->io_wq->chann)); + if (!tmp) { + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "8353 error kmalloc memory for HDWQ " + "%d %s\n", + lpfc_buf->hdwq_no, __func__); + return NULL; + } + + tmp->dma_sgl = dma_pool_alloc(phba->lpfc_sg_dma_buf_pool, + GFP_ATOMIC, &tmp->dma_phys_sgl); + if (!tmp->dma_sgl) { + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "8354 error pool_alloc memory for HDWQ " + "%d %s\n", + lpfc_buf->hdwq_no, __func__); + kfree(tmp); + return NULL; + } + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + list_add_tail(&tmp->list_node, &lpfc_buf->dma_sgl_xtra_list); + } + + allocated_sgl = list_last_entry(&lpfc_buf->dma_sgl_xtra_list, + struct sli4_hybrid_sgl, + list_node); + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + + return allocated_sgl; +} + +/** + * lpfc_put_sgl_per_hdwq - Put one SGL chunk into hdwq pool + * @phba: The HBA for which this call is being executed. + * @lpfc_buf: IO buf structure with the SGL chunk + * + * This routine puts one SGL chunk buffer into hdwq's SGL chunk pool. + * + * Return codes: + * 0 - Success + * -EINVAL - Error + **/ +int +lpfc_put_sgl_per_hdwq(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_buf) +{ + int rc = 0; + struct sli4_hybrid_sgl *list_entry = NULL; + struct sli4_hybrid_sgl *tmp = NULL; + struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq; + struct list_head *buf_list = &hdwq->sgl_list; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + if (likely(!list_empty(&lpfc_buf->dma_sgl_xtra_list))) { + list_for_each_entry_safe(list_entry, tmp, + &lpfc_buf->dma_sgl_xtra_list, + list_node) { + list_move_tail(&list_entry->list_node, + buf_list); + } + } else { + rc = -EINVAL; + } + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + return rc; +} + +/** + * lpfc_free_sgl_per_hdwq - Free all SGL chunks of hdwq pool + * @phba: phba object + * @hdwq: hdwq to cleanup sgl buff resources on + * + * This routine frees all SGL chunks of hdwq SGL chunk pool. + * + * Return codes: + * None + **/ +void +lpfc_free_sgl_per_hdwq(struct lpfc_hba *phba, + struct lpfc_sli4_hdw_queue *hdwq) +{ + struct list_head *buf_list = &hdwq->sgl_list; + struct sli4_hybrid_sgl *list_entry = NULL; + struct sli4_hybrid_sgl *tmp = NULL; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + /* Free sgl pool */ + list_for_each_entry_safe(list_entry, tmp, + buf_list, list_node) { + dma_pool_free(phba->lpfc_sg_dma_buf_pool, + list_entry->dma_sgl, + list_entry->dma_phys_sgl); + list_del(&list_entry->list_node); + kfree(list_entry); + } + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); +} + +/** + * lpfc_get_cmd_rsp_buf_per_hdwq - Get one CMD/RSP buffer from hdwq + * @phba: The HBA for which this call is being executed. + * @lpfc_buf: IO buf structure to attach the CMD/RSP buffer + * + * This routine gets one CMD/RSP buffer from hdwq's CMD/RSP pool, + * and will allocate an CMD/RSP buffer if the pool is empty. + * + * Return codes: + * NULL - Error + * Pointer to fcp_cmd_rsp_buf - Success + **/ +struct fcp_cmd_rsp_buf * +lpfc_get_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba, + struct lpfc_io_buf *lpfc_buf) +{ + struct fcp_cmd_rsp_buf *list_entry = NULL; + struct fcp_cmd_rsp_buf *tmp = NULL; + struct fcp_cmd_rsp_buf *allocated_buf = NULL; + struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq; + struct list_head *buf_list = &hdwq->cmd_rsp_buf_list; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + if (likely(!list_empty(buf_list))) { + /* break off 1 chunk from the list */ + list_for_each_entry_safe(list_entry, tmp, + buf_list, + list_node) { + list_move_tail(&list_entry->list_node, + &lpfc_buf->dma_cmd_rsp_list); + break; + } + } else { + /* allocate more */ + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + tmp = kmalloc_node(sizeof(*tmp), GFP_ATOMIC, + cpu_to_node(hdwq->io_wq->chann)); + if (!tmp) { + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "8355 error kmalloc memory for HDWQ " + "%d %s\n", + lpfc_buf->hdwq_no, __func__); + return NULL; + } + + tmp->fcp_cmnd = dma_pool_alloc(phba->lpfc_cmd_rsp_buf_pool, + GFP_ATOMIC, + &tmp->fcp_cmd_rsp_dma_handle); + + if (!tmp->fcp_cmnd) { + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "8356 error pool_alloc memory for HDWQ " + "%d %s\n", + lpfc_buf->hdwq_no, __func__); + kfree(tmp); + return NULL; + } + + tmp->fcp_rsp = (struct fcp_rsp *)((uint8_t *)tmp->fcp_cmnd + + sizeof(struct fcp_cmnd)); + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + list_add_tail(&tmp->list_node, &lpfc_buf->dma_cmd_rsp_list); + } + + allocated_buf = list_last_entry(&lpfc_buf->dma_cmd_rsp_list, + struct fcp_cmd_rsp_buf, + list_node); + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + + return allocated_buf; +} + +/** + * lpfc_put_cmd_rsp_buf_per_hdwq - Put one CMD/RSP buffer into hdwq pool + * @phba: The HBA for which this call is being executed. + * @lpfc_buf: IO buf structure with the CMD/RSP buf + * + * This routine puts one CMD/RSP buffer into executing CPU's CMD/RSP pool. + * + * Return codes: + * 0 - Success + * -EINVAL - Error + **/ +int +lpfc_put_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba, + struct lpfc_io_buf *lpfc_buf) +{ + int rc = 0; + struct fcp_cmd_rsp_buf *list_entry = NULL; + struct fcp_cmd_rsp_buf *tmp = NULL; + struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq; + struct list_head *buf_list = &hdwq->cmd_rsp_buf_list; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + if (likely(!list_empty(&lpfc_buf->dma_cmd_rsp_list))) { + list_for_each_entry_safe(list_entry, tmp, + &lpfc_buf->dma_cmd_rsp_list, + list_node) { + list_move_tail(&list_entry->list_node, + buf_list); + } + } else { + rc = -EINVAL; + } + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + return rc; +} + +/** + * lpfc_free_cmd_rsp_buf_per_hdwq - Free all CMD/RSP chunks of hdwq pool + * @phba: phba object + * @hdwq: hdwq to cleanup cmd rsp buff resources on + * + * This routine frees all CMD/RSP buffers of hdwq's CMD/RSP buf pool. + * + * Return codes: + * None + **/ +void +lpfc_free_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba, + struct lpfc_sli4_hdw_queue *hdwq) +{ + struct list_head *buf_list = &hdwq->cmd_rsp_buf_list; + struct fcp_cmd_rsp_buf *list_entry = NULL; + struct fcp_cmd_rsp_buf *tmp = NULL; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + /* Free cmd_rsp buf pool */ + list_for_each_entry_safe(list_entry, tmp, + buf_list, + list_node) { + dma_pool_free(phba->lpfc_cmd_rsp_buf_pool, + list_entry->fcp_cmnd, + list_entry->fcp_cmd_rsp_dma_handle); + list_del(&list_entry->list_node); + kfree(list_entry); + } + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); +} |
