diff options
Diffstat (limited to 'drivers/scsi/ufs')
| -rw-r--r-- | drivers/scsi/ufs/ufs.h | 155 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd-pci.c | 99 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd-pltfrm.c | 57 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd.c | 1170 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd.h | 59 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshci.h | 2 |
6 files changed, 1376 insertions, 166 deletions
diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h index 139bc0647b41..bce09a6898c4 100644 --- a/drivers/scsi/ufs/ufs.h +++ b/drivers/scsi/ufs/ufs.h @@ -36,10 +36,17 @@ #ifndef _UFS_H #define _UFS_H +#include <linux/mutex.h> +#include <linux/types.h> + #define MAX_CDB_SIZE 16 +#define GENERAL_UPIU_REQUEST_SIZE 32 +#define QUERY_DESC_MAX_SIZE 256 +#define QUERY_OSF_SIZE (GENERAL_UPIU_REQUEST_SIZE - \ + (sizeof(struct utp_upiu_header))) #define UPIU_HEADER_DWORD(byte3, byte2, byte1, byte0)\ - ((byte3 << 24) | (byte2 << 16) |\ + cpu_to_be32((byte3 << 24) | (byte2 << 16) |\ (byte1 << 8) | (byte0)) /* @@ -62,7 +69,7 @@ enum { UPIU_TRANSACTION_COMMAND = 0x01, UPIU_TRANSACTION_DATA_OUT = 0x02, UPIU_TRANSACTION_TASK_REQ = 0x04, - UPIU_TRANSACTION_QUERY_REQ = 0x26, + UPIU_TRANSACTION_QUERY_REQ = 0x16, }; /* UTP UPIU Transaction Codes Target to Initiator */ @@ -73,6 +80,7 @@ enum { UPIU_TRANSACTION_TASK_RSP = 0x24, UPIU_TRANSACTION_READY_XFER = 0x31, UPIU_TRANSACTION_QUERY_RSP = 0x36, + UPIU_TRANSACTION_REJECT_UPIU = 0x3F, }; /* UPIU Read/Write flags */ @@ -90,8 +98,41 @@ enum { UPIU_TASK_ATTR_ACA = 0x03, }; -/* UTP QUERY Transaction Specific Fields OpCode */ +/* UPIU Query request function */ enum { + UPIU_QUERY_FUNC_STANDARD_READ_REQUEST = 0x01, + UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST = 0x81, +}; + +/* Flag idn for Query Requests*/ +enum flag_idn { + QUERY_FLAG_IDN_FDEVICEINIT = 0x01, + QUERY_FLAG_IDN_BKOPS_EN = 0x04, +}; + +/* Attribute idn for Query requests */ +enum attr_idn { + QUERY_ATTR_IDN_BKOPS_STATUS = 0x05, + QUERY_ATTR_IDN_EE_CONTROL = 0x0D, + QUERY_ATTR_IDN_EE_STATUS = 0x0E, +}; + +/* Exception event mask values */ +enum { + MASK_EE_STATUS = 0xFFFF, + MASK_EE_URGENT_BKOPS = (1 << 2), +}; + +/* Background operation status */ +enum { + BKOPS_STATUS_NO_OP = 0x0, + BKOPS_STATUS_NON_CRITICAL = 0x1, + BKOPS_STATUS_PERF_IMPACT = 0x2, + BKOPS_STATUS_CRITICAL = 0x3, +}; + +/* UTP QUERY Transaction Specific Fields OpCode */ +enum query_opcode { UPIU_QUERY_OPCODE_NOP = 0x0, UPIU_QUERY_OPCODE_READ_DESC = 0x1, UPIU_QUERY_OPCODE_WRITE_DESC = 0x2, @@ -103,6 +144,21 @@ enum { UPIU_QUERY_OPCODE_TOGGLE_FLAG = 0x8, }; +/* Query response result code */ +enum { + QUERY_RESULT_SUCCESS = 0x00, + QUERY_RESULT_NOT_READABLE = 0xF6, + QUERY_RESULT_NOT_WRITEABLE = 0xF7, + QUERY_RESULT_ALREADY_WRITTEN = 0xF8, + QUERY_RESULT_INVALID_LENGTH = 0xF9, + QUERY_RESULT_INVALID_VALUE = 0xFA, + QUERY_RESULT_INVALID_SELECTOR = 0xFB, + QUERY_RESULT_INVALID_INDEX = 0xFC, + QUERY_RESULT_INVALID_IDN = 0xFD, + QUERY_RESULT_INVALID_OPCODE = 0xFE, + QUERY_RESULT_GENERAL_FAILURE = 0xFF, +}; + /* UTP Transfer Request Command Type (CT) */ enum { UPIU_COMMAND_SET_TYPE_SCSI = 0x0, @@ -110,10 +166,18 @@ enum { UPIU_COMMAND_SET_TYPE_QUERY = 0x2, }; +/* UTP Transfer Request Command Offset */ +#define UPIU_COMMAND_TYPE_OFFSET 28 + +/* Offset of the response code in the UPIU header */ +#define UPIU_RSP_CODE_OFFSET 8 + enum { - MASK_SCSI_STATUS = 0xFF, - MASK_TASK_RESPONSE = 0xFF00, - MASK_RSP_UPIU_RESULT = 0xFFFF, + MASK_SCSI_STATUS = 0xFF, + MASK_TASK_RESPONSE = 0xFF00, + MASK_RSP_UPIU_RESULT = 0xFFFF, + MASK_QUERY_DATA_SEG_LEN = 0xFFFF, + MASK_RSP_EXCEPTION_EVENT = 0x10000, }; /* Task management service response */ @@ -138,26 +202,59 @@ struct utp_upiu_header { /** * struct utp_upiu_cmd - Command UPIU structure - * @header: UPIU header structure DW-0 to DW-2 * @data_transfer_len: Data Transfer Length DW-3 * @cdb: Command Descriptor Block CDB DW-4 to DW-7 */ struct utp_upiu_cmd { - struct utp_upiu_header header; u32 exp_data_transfer_len; u8 cdb[MAX_CDB_SIZE]; }; /** - * struct utp_upiu_rsp - Response UPIU structure - * @header: UPIU header DW-0 to DW-2 + * struct utp_upiu_query - upiu request buffer structure for + * query request. + * @opcode: command to perform B-0 + * @idn: a value that indicates the particular type of data B-1 + * @index: Index to further identify data B-2 + * @selector: Index to further identify data B-3 + * @reserved_osf: spec reserved field B-4,5 + * @length: number of descriptor bytes to read/write B-6,7 + * @value: Attribute value to be written DW-5 + * @reserved: spec reserved DW-6,7 + */ +struct utp_upiu_query { + u8 opcode; + u8 idn; + u8 index; + u8 selector; + u16 reserved_osf; + u16 length; + u32 value; + u32 reserved[2]; +}; + +/** + * struct utp_upiu_req - general upiu request structure + * @header:UPIU header structure DW-0 to DW-2 + * @sc: fields structure for scsi command DW-3 to DW-7 + * @qr: fields structure for query request DW-3 to DW-7 + */ +struct utp_upiu_req { + struct utp_upiu_header header; + union { + struct utp_upiu_cmd sc; + struct utp_upiu_query qr; + }; +}; + +/** + * struct utp_cmd_rsp - Response UPIU structure * @residual_transfer_count: Residual transfer count DW-3 * @reserved: Reserved double words DW-4 to DW-7 * @sense_data_len: Sense data length DW-8 U16 * @sense_data: Sense data field DW-8 to DW-12 */ -struct utp_upiu_rsp { - struct utp_upiu_header header; +struct utp_cmd_rsp { u32 residual_transfer_count; u32 reserved[4]; u16 sense_data_len; @@ -165,6 +262,20 @@ struct utp_upiu_rsp { }; /** + * struct utp_upiu_rsp - general upiu response structure + * @header: UPIU header structure DW-0 to DW-2 + * @sr: fields structure for scsi command DW-3 to DW-12 + * @qr: fields structure for query request DW-3 to DW-7 + */ +struct utp_upiu_rsp { + struct utp_upiu_header header; + union { + struct utp_cmd_rsp sr; + struct utp_upiu_query qr; + }; +}; + +/** * struct utp_upiu_task_req - Task request UPIU structure * @header - UPIU header structure DW0 to DW-2 * @input_param1: Input parameter 1 DW-3 @@ -194,4 +305,24 @@ struct utp_upiu_task_rsp { u32 reserved[3]; }; +/** + * struct ufs_query_req - parameters for building a query request + * @query_func: UPIU header query function + * @upiu_req: the query request data + */ +struct ufs_query_req { + u8 query_func; + struct utp_upiu_query upiu_req; +}; + +/** + * struct ufs_query_resp - UPIU QUERY + * @response: device response code + * @upiu_res: query response data + */ +struct ufs_query_res { + u8 response; + struct utp_upiu_query upiu_res; +}; + #endif /* End of Header */ diff --git a/drivers/scsi/ufs/ufshcd-pci.c b/drivers/scsi/ufs/ufshcd-pci.c index 48be39a6f6d7..a823cf44e949 100644 --- a/drivers/scsi/ufs/ufshcd-pci.c +++ b/drivers/scsi/ufs/ufshcd-pci.c @@ -35,6 +35,7 @@ #include "ufshcd.h" #include <linux/pci.h> +#include <linux/pm_runtime.h> #ifdef CONFIG_PM /** @@ -44,7 +45,7 @@ * * Returns -ENOSYS */ -static int ufshcd_pci_suspend(struct pci_dev *pdev, pm_message_t state) +static int ufshcd_pci_suspend(struct device *dev) { /* * TODO: @@ -61,7 +62,7 @@ static int ufshcd_pci_suspend(struct pci_dev *pdev, pm_message_t state) * * Returns -ENOSYS */ -static int ufshcd_pci_resume(struct pci_dev *pdev) +static int ufshcd_pci_resume(struct device *dev) { /* * TODO: @@ -71,8 +72,45 @@ static int ufshcd_pci_resume(struct pci_dev *pdev) return -ENOSYS; } +#else +#define ufshcd_pci_suspend NULL +#define ufshcd_pci_resume NULL #endif /* CONFIG_PM */ +#ifdef CONFIG_PM_RUNTIME +static int ufshcd_pci_runtime_suspend(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!hba) + return 0; + + return ufshcd_runtime_suspend(hba); +} +static int ufshcd_pci_runtime_resume(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!hba) + return 0; + + return ufshcd_runtime_resume(hba); +} +static int ufshcd_pci_runtime_idle(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!hba) + return 0; + + return ufshcd_runtime_idle(hba); +} +#else /* !CONFIG_PM_RUNTIME */ +#define ufshcd_pci_runtime_suspend NULL +#define ufshcd_pci_runtime_resume NULL +#define ufshcd_pci_runtime_idle NULL +#endif /* CONFIG_PM_RUNTIME */ + /** * ufshcd_pci_shutdown - main function to put the controller in reset state * @pdev: pointer to PCI device handle @@ -91,12 +129,10 @@ static void ufshcd_pci_remove(struct pci_dev *pdev) { struct ufs_hba *hba = pci_get_drvdata(pdev); - disable_irq(pdev->irq); + pm_runtime_forbid(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); ufshcd_remove(hba); - pci_release_regions(pdev); pci_set_drvdata(pdev, NULL); - pci_clear_master(pdev); - pci_disable_device(pdev); } /** @@ -133,55 +169,49 @@ ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) void __iomem *mmio_base; int err; - err = pci_enable_device(pdev); + err = pcim_enable_device(pdev); if (err) { - dev_err(&pdev->dev, "pci_enable_device failed\n"); - goto out_error; + dev_err(&pdev->dev, "pcim_enable_device failed\n"); + return err; } pci_set_master(pdev); - - err = pci_request_regions(pdev, UFSHCD); + err = pcim_iomap_regions(pdev, 1 << 0, UFSHCD); if (err < 0) { - dev_err(&pdev->dev, "request regions failed\n"); - goto out_disable; + dev_err(&pdev->dev, "request and iomap failed\n"); + return err; } - mmio_base = pci_ioremap_bar(pdev, 0); - if (!mmio_base) { - dev_err(&pdev->dev, "memory map failed\n"); - err = -ENOMEM; - goto out_release_regions; - } + mmio_base = pcim_iomap_table(pdev)[0]; err = ufshcd_set_dma_mask(pdev); if (err) { dev_err(&pdev->dev, "set dma mask failed\n"); - goto out_iounmap; + return err; } err = ufshcd_init(&pdev->dev, &hba, mmio_base, pdev->irq); if (err) { dev_err(&pdev->dev, "Initialization failed\n"); - goto out_iounmap; + return err; } pci_set_drvdata(pdev, hba); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_allow(&pdev->dev); return 0; - -out_iounmap: - iounmap(mmio_base); -out_release_regions: - pci_release_regions(pdev); -out_disable: - pci_clear_master(pdev); - pci_disable_device(pdev); -out_error: - return err; } +static const struct dev_pm_ops ufshcd_pci_pm_ops = { + .suspend = ufshcd_pci_suspend, + .resume = ufshcd_pci_resume, + .runtime_suspend = ufshcd_pci_runtime_suspend, + .runtime_resume = ufshcd_pci_runtime_resume, + .runtime_idle = ufshcd_pci_runtime_idle, +}; + static DEFINE_PCI_DEVICE_TABLE(ufshcd_pci_tbl) = { { PCI_VENDOR_ID_SAMSUNG, 0xC00C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, { } /* terminate list */ @@ -195,10 +225,9 @@ static struct pci_driver ufshcd_pci_driver = { .probe = ufshcd_pci_probe, .remove = ufshcd_pci_remove, .shutdown = ufshcd_pci_shutdown, -#ifdef CONFIG_PM - .suspend = ufshcd_pci_suspend, - .resume = ufshcd_pci_resume, -#endif + .driver = { + .pm = &ufshcd_pci_pm_ops + }, }; module_pci_driver(ufshcd_pci_driver); diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.c b/drivers/scsi/ufs/ufshcd-pltfrm.c index c42db40d4e51..5e4623225422 100644 --- a/drivers/scsi/ufs/ufshcd-pltfrm.c +++ b/drivers/scsi/ufs/ufshcd-pltfrm.c @@ -34,6 +34,7 @@ */ #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include "ufshcd.h" @@ -87,6 +88,40 @@ static int ufshcd_pltfrm_resume(struct device *dev) #define ufshcd_pltfrm_resume NULL #endif +#ifdef CONFIG_PM_RUNTIME +static int ufshcd_pltfrm_runtime_suspend(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!hba) + return 0; + + return ufshcd_runtime_suspend(hba); +} +static int ufshcd_pltfrm_runtime_resume(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!hba) + return 0; + + return ufshcd_runtime_resume(hba); +} +static int ufshcd_pltfrm_runtime_idle(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!hba) + return 0; + + return ufshcd_runtime_idle(hba); +} +#else /* !CONFIG_PM_RUNTIME */ +#define ufshcd_pltfrm_runtime_suspend NULL +#define ufshcd_pltfrm_runtime_resume NULL +#define ufshcd_pltfrm_runtime_idle NULL +#endif /* CONFIG_PM_RUNTIME */ + /** * ufshcd_pltfrm_probe - probe routine of the driver * @pdev: pointer to Platform device handle @@ -102,15 +137,8 @@ static int ufshcd_pltfrm_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!mem_res) { - dev_err(dev, "Memory resource not available\n"); - err = -ENODEV; - goto out; - } - mmio_base = devm_ioremap_resource(dev, mem_res); if (IS_ERR(mmio_base)) { - dev_err(dev, "memory map failed\n"); err = PTR_ERR(mmio_base); goto out; } @@ -122,14 +150,22 @@ static int ufshcd_pltfrm_probe(struct platform_device *pdev) goto out; } + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + err = ufshcd_init(dev, &hba, mmio_base, irq); if (err) { dev_err(dev, "Intialization failed\n"); - goto out; + goto out_disable_rpm; } platform_set_drvdata(pdev, hba); + return 0; + +out_disable_rpm: + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); out: return err; } @@ -144,7 +180,7 @@ static int ufshcd_pltfrm_remove(struct platform_device *pdev) { struct ufs_hba *hba = platform_get_drvdata(pdev); - disable_irq(hba->irq); + pm_runtime_get_sync(&(pdev)->dev); ufshcd_remove(hba); return 0; } @@ -157,6 +193,9 @@ static const struct of_device_id ufs_of_match[] = { static const struct dev_pm_ops ufshcd_dev_pm_ops = { .suspend = ufshcd_pltfrm_suspend, .resume = ufshcd_pltfrm_resume, + .runtime_suspend = ufshcd_pltfrm_runtime_suspend, + .runtime_resume = ufshcd_pltfrm_runtime_resume, + .runtime_idle = ufshcd_pltfrm_runtime_idle, }; static struct platform_driver ufshcd_pltfrm_driver = { diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index b743bd6fce6b..b36ca9a2dfbb 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -43,6 +43,19 @@ /* UIC command timeout, unit: ms */ #define UIC_CMD_TIMEOUT 500 +/* NOP OUT retries waiting for NOP IN response */ +#define NOP_OUT_RETRIES 10 +/* Timeout after 30 msecs if NOP OUT hangs without response */ +#define NOP_OUT_TIMEOUT 30 /* msecs */ + +/* Query request retries */ +#define QUERY_REQ_RETRIES 10 +/* Query request timeout */ +#define QUERY_REQ_TIMEOUT 30 /* msec */ + +/* Expose the flag value from utp_upiu_query.value */ +#define MASK_QUERY_UPIU_FLAG_LOC 0xFF + enum { UFSHCD_MAX_CHANNEL = 0, UFSHCD_MAX_ID = 1, @@ -71,6 +84,40 @@ enum { INT_AGGR_CONFIG, }; +/* + * ufshcd_wait_for_register - wait for register value to change + * @hba - per-adapter interface + * @reg - mmio register offset + * @mask - mask to apply to read register value + * @val - wait condition + * @interval_us - polling interval in microsecs + * @timeout_ms - timeout in millisecs + * + * Returns -ETIMEDOUT on error, zero on success + */ +static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, + u32 val, unsigned long interval_us, unsigned long timeout_ms) +{ + int err = 0; + unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + + /* ignore bits that we don't intend to wait on */ + val = val & mask; + + while ((ufshcd_readl(hba, reg) & mask) != val) { + /* wakeup within 50us of expiry */ + usleep_range(interval_us, interval_us + 50); + + if (time_after(jiffies, timeout)) { + if ((ufshcd_readl(hba, reg) & mask) != val) + err = -ETIMEDOUT; + break; + } + } + + return err; +} + /** * ufshcd_get_intr_mask - Get the interrupt bit mask * @hba - Pointer to adapter instance @@ -191,18 +238,13 @@ static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) } /** - * ufshcd_is_valid_req_rsp - checks if controller TR response is valid + * ufshcd_get_req_rsp - returns the TR response transaction type * @ucd_rsp_ptr: pointer to response UPIU - * - * This function checks the response UPIU for valid transaction type in - * response field - * Returns 0 on success, non-zero on failure */ static inline int -ufshcd_is_valid_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) +ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) { - return ((be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24) == - UPIU_TRANSACTION_RESPONSE) ? 0 : DID_ERROR << 16; + return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; } /** @@ -219,6 +261,21 @@ ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr) } /** + * ufshcd_is_exception_event - Check if the device raised an exception event + * @ucd_rsp_ptr: pointer to response UPIU + * + * The function checks if the device raised an exception event indicated in + * the Device Information field of response UPIU. + * + * Returns true if exception is raised, false otherwise. + */ +static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) +{ + return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & + MASK_RSP_EXCEPTION_EVENT ? true : false; +} + +/** * ufshcd_config_int_aggr - Configure interrupt aggregation values. * Currently there is no use case where we want to configure * interrupt aggregation dynamically. So to configure interrupt @@ -299,14 +356,68 @@ static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) { int len; if (lrbp->sense_buffer) { - len = be16_to_cpu(lrbp->ucd_rsp_ptr->sense_data_len); + len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); memcpy(lrbp->sense_buffer, - lrbp->ucd_rsp_ptr->sense_data, + lrbp->ucd_rsp_ptr->sr.sense_data, min_t(int, len, SCSI_SENSE_BUFFERSIZE)); } } /** + * ufshcd_query_to_cpu() - formats the buffer to native cpu endian + * @response: upiu query response to convert + */ +static inline void ufshcd_query_to_cpu(struct utp_upiu_query *response) +{ + response->length = be16_to_cpu(response->length); + response->value = be32_to_cpu(response->value); +} + +/** + * ufshcd_query_to_be() - formats the buffer to big endian + * @request: upiu query request to convert + */ +static inline void ufshcd_query_to_be(struct utp_upiu_query *request) +{ + request->length = cpu_to_be16(request->length); + request->value = cpu_to_be32(request->value); +} + +/** + * ufshcd_copy_query_response() - Copy the Query Response and the data + * descriptor + * @hba: per adapter instance + * @lrb - pointer to local reference block + */ +static +void ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +{ + struct ufs_query_res *query_res = &hba->dev_cmd.query.response; + + /* Get the UPIU response */ + query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >> + UPIU_RSP_CODE_OFFSET; + + memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); + ufshcd_query_to_cpu(&query_res->upiu_res); + + + /* Get the descriptor */ + if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { + u8 *descp = (u8 *)&lrbp->ucd_rsp_ptr + + GENERAL_UPIU_REQUEST_SIZE; + u16 len; + + /* data segment length */ + len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & + MASK_QUERY_DATA_SEG_LEN; + + memcpy(hba->dev_cmd.query.descriptor, descp, + min_t(u16, len, QUERY_DESC_MAX_SIZE)); + } +} + +/** * ufshcd_hba_capabilities - Read controller capabilities * @hba: per adapter instance */ @@ -519,76 +630,170 @@ static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) } /** + * ufshcd_prepare_req_desc_hdr() - Fills the requests header + * descriptor according to request + * @lrbp: pointer to local reference block + * @upiu_flags: flags required in the header + * @cmd_dir: requests data direction + */ +static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, + u32 *upiu_flags, enum dma_data_direction cmd_dir) +{ + struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; + u32 data_direction; + u32 dword_0; + + if (cmd_dir == DMA_FROM_DEVICE) { + data_direction = UTP_DEVICE_TO_HOST; + *upiu_flags = UPIU_CMD_FLAGS_READ; + } else if (cmd_dir == DMA_TO_DEVICE) { + data_direction = UTP_HOST_TO_DEVICE; + *upiu_flags = UPIU_CMD_FLAGS_WRITE; + } else { + data_direction = UTP_NO_DATA_TRANSFER; + *upiu_flags = UPIU_CMD_FLAGS_NONE; + } + + dword_0 = data_direction | (lrbp->command_type + << UPIU_COMMAND_TYPE_OFFSET); + if (lrbp->intr_cmd) + dword_0 |= UTP_REQ_DESC_INT_CMD; + + /* Transfer request descriptor header fields */ + req_desc->header.dword_0 = cpu_to_le32(dword_0); + + /* + * assigning invalid value for command status. Controller + * updates OCS on command completion, with the command + * status + */ + req_desc->header.dword_2 = + cpu_to_le32(OCS_INVALID_COMMAND_STATUS); +} + +/** + * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, + * for scsi commands + * @lrbp - local reference block pointer + * @upiu_flags - flags + */ +static +void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags) +{ + struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; + + /* command descriptor fields */ + ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( + UPIU_TRANSACTION_COMMAND, upiu_flags, + lrbp->lun, lrbp->task_tag); + ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( + UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0); + + /* Total EHS length and Data segment length will be zero */ + ucd_req_ptr->header.dword_2 = 0; + + ucd_req_ptr->sc.exp_data_transfer_len = + cpu_to_be32(lrbp->cmd->sdb.length); + + memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, + (min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE))); +} + +/** + * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc, + * for query requsts + * @hba: UFS hba + * @lrbp: local reference block pointer + * @upiu_flags: flags + */ +static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, + struct ufshcd_lrb *lrbp, u32 upiu_flags) +{ + struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; + struct ufs_query *query = &hba->dev_cmd.query; + u16 len = query->request.upiu_req.length; + u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE; + + /* Query request header */ + ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( + UPIU_TRANSACTION_QUERY_REQ, upiu_flags, + lrbp->lun, lrbp->task_tag); + ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( + 0, query->request.query_func, 0, 0); + + /* Data segment length */ + ucd_req_ptr->header.dword_2 = UPIU_HEADER_DWORD( + 0, 0, len >> 8, (u8)len); + + /* Copy the Query Request buffer as is */ + memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, + QUERY_OSF_SIZE); + ufshcd_query_to_be(&ucd_req_ptr->qr); + + /* Copy the Descriptor */ + if ((len > 0) && (query->request.upiu_req.opcode == + UPIU_QUERY_OPCODE_WRITE_DESC)) { + memcpy(descp, query->descriptor, + min_t(u16, len, QUERY_DESC_MAX_SIZE)); + } +} + +static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) +{ + struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; + + memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); + + /* command descriptor fields */ + ucd_req_ptr->header.dword_0 = + UPIU_HEADER_DWORD( + UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag); +} + +/** * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU) + * @hba - per adapter instance * @lrb - pointer to local reference block */ -static void ufshcd_compose_upiu(struct ufshcd_lrb *lrbp) +static int ufshcd_compose_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { - struct utp_transfer_req_desc *req_desc; - struct utp_upiu_cmd *ucd_cmd_ptr; - u32 data_direction; u32 upiu_flags; - - ucd_cmd_ptr = lrbp->ucd_cmd_ptr; - req_desc = lrbp->utr_descriptor_ptr; + int ret = 0; switch (lrbp->command_type) { case UTP_CMD_TYPE_SCSI: - if (lrbp->cmd->sc_data_direction == DMA_FROM_DEVICE) { - data_direction = UTP_DEVICE_TO_HOST; - upiu_flags = UPIU_CMD_FLAGS_READ; - } else if (lrbp->cmd->sc_data_direction == DMA_TO_DEVICE) { - data_direction = UTP_HOST_TO_DEVICE; - upiu_flags = UPIU_CMD_FLAGS_WRITE; + if (likely(lrbp->cmd)) { + ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, + lrbp->cmd->sc_data_direction); + ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); } else { - data_direction = UTP_NO_DATA_TRANSFER; - upiu_flags = UPIU_CMD_FLAGS_NONE; + ret = -EINVAL; } - - /* Transfer request descriptor header fields */ - req_desc->header.dword_0 = - cpu_to_le32(data_direction | UTP_SCSI_COMMAND); - - /* - * assigning invalid value for command status. Controller - * updates OCS on command completion, with the command - * status - */ - req_desc->header.dword_2 = - cpu_to_le32(OCS_INVALID_COMMAND_STATUS); - - /* command descriptor fields */ - ucd_cmd_ptr->header.dword_0 = - cpu_to_be32(UPIU_HEADER_DWORD(UPIU_TRANSACTION_COMMAND, - upiu_flags, - lrbp->lun, - lrbp->task_tag)); - ucd_cmd_ptr->header.dword_1 = - cpu_to_be32( - UPIU_HEADER_DWORD(UPIU_COMMAND_SET_TYPE_SCSI, - 0, - 0, - 0)); - - /* Total EHS length and Data segment length will be zero */ - ucd_cmd_ptr->header.dword_2 = 0; - - ucd_cmd_ptr->exp_data_transfer_len = - cpu_to_be32(lrbp->cmd->sdb.length); - - memcpy(ucd_cmd_ptr->cdb, - lrbp->cmd->cmnd, - (min_t(unsigned short, - lrbp->cmd->cmd_len, - MAX_CDB_SIZE))); break; case UTP_CMD_TYPE_DEV_MANAGE: - /* For query function implementation */ + ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE); + if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) + ufshcd_prepare_utp_query_req_upiu( + hba, lrbp, upiu_flags); + else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) + ufshcd_prepare_utp_nop_upiu(lrbp); + else + ret = -EINVAL; break; case UTP_CMD_TYPE_UFS: /* For UFS native command implementation */ + ret = -ENOTSUPP; + dev_err(hba->dev, "%s: UFS native command are not supported\n", + __func__); + break; + default: + ret = -ENOTSUPP; + dev_err(hba->dev, "%s: unknown command type: 0x%x\n", + __func__, lrbp->command_type); break; } /* end of switch */ + + return ret; } /** @@ -615,21 +820,37 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) goto out; } + /* acquire the tag to make sure device cmds don't use it */ + if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) { + /* + * Dev manage command in progress, requeue the command. + * Requeuing the command helps in cases where the request *may* + * find different tag instead of waiting for dev manage command + * completion. + */ + err = SCSI_MLQUEUE_HOST_BUSY; + goto out; + } + lrbp = &hba->lrb[tag]; + WARN_ON(lrbp->cmd); lrbp->cmd = cmd; lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE; lrbp->sense_buffer = cmd->sense_buffer; lrbp->task_tag = tag; lrbp->lun = cmd->device->lun; - + lrbp->intr_cmd = false; lrbp->command_type = UTP_CMD_TYPE_SCSI; /* form UPIU before issuing the command */ - ufshcd_compose_upiu(lrbp); + ufshcd_compose_upiu(hba, lrbp); err = ufshcd_map_sg(lrbp); - if (err) + if (err) { + lrbp->cmd = NULL; + clear_bit_unlock(tag, &hba->lrb_in_use); goto out; + } /* issue command to the controller */ spin_lock_irqsave(hba->host->host_lock, flags); @@ -639,6 +860,338 @@ out: return err; } +static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, + struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) +{ + lrbp->cmd = NULL; + lrbp->sense_bufflen = 0; + lrbp->sense_buffer = NULL; + lrbp->task_tag = tag; + lrbp->lun = 0; /* device management cmd is not specific to any LUN */ + lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; + lrbp->intr_cmd = true; /* No interrupt aggregation */ + hba->dev_cmd.type = cmd_type; + + return ufshcd_compose_upiu(hba, lrbp); +} + +static int +ufshcd_clear_cmd(struct ufs_hba *hba, int tag) +{ + int err = 0; + unsigned long flags; + u32 mask = 1 << tag; + + /* clear outstanding transaction before retry */ + spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_utrl_clear(hba, tag); + spin_unlock_irqrestore(hba->host->host_lock, flags); + + /* + * wait for for h/w to clear corresponding bit in door-bell. + * max. wait is 1 sec. + */ + err = ufshcd_wait_for_register(hba, + REG_UTP_TRANSFER_REQ_DOOR_BELL, + mask, ~mask, 1000, 1000); + + return err; +} + +/** + * ufshcd_dev_cmd_completion() - handles device management command responses + * @hba: per adapter instance + * @lrbp: pointer to local reference block + */ +static int +ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +{ + int resp; + int err = 0; + + resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); + + switch (resp) { + case UPIU_TRANSACTION_NOP_IN: + if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) { + err = -EINVAL; + dev_err(hba->dev, "%s: unexpected response %x\n", + __func__, resp); + } + break; + case UPIU_TRANSACTION_QUERY_RSP: + ufshcd_copy_query_response(hba, lrbp); + break; + case UPIU_TRANSACTION_REJECT_UPIU: + /* TODO: handle Reject UPIU Response */ + err = -EPERM; + dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", + __func__); + break; + default: + err = -EINVAL; + dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", + __func__, resp); + break; + } + + return err; +} + +static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, + struct ufshcd_lrb *lrbp, int max_timeout) +{ + int err = 0; + unsigned long time_left; + unsigned long flags; + + time_left = wait_for_completion_timeout(hba->dev_cmd.complete, + msecs_to_jiffies(max_timeout)); + + spin_lock_irqsave(hba->host->host_lock, flags); + hba->dev_cmd.complete = NULL; + if (likely(time_left)) { + err = ufshcd_get_tr_ocs(lrbp); + if (!err) + err = ufshcd_dev_cmd_completion(hba, lrbp); + } + spin_unlock_irqrestore(hba->host->host_lock, flags); + + if (!time_left) { + err = -ETIMEDOUT; + if (!ufshcd_clear_cmd(hba, lrbp->task_tag)) + /* sucessfully cleared the command, retry if needed */ + err = -EAGAIN; + } + + return err; +} + +/** + * ufshcd_get_dev_cmd_tag - Get device management command tag + * @hba: per-adapter instance + * @tag: pointer to variable with available slot value + * + * Get a free slot and lock it until device management command + * completes. + * + * Returns false if free slot is unavailable for locking, else + * return true with tag value in @tag. + */ +static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out) +{ + int tag; + bool ret = false; + unsigned long tmp; + + if (!tag_out) + goto out; + + do { + tmp = ~hba->lrb_in_use; + tag = find_last_bit(&tmp, hba->nutrs); + if (tag >= hba->nutrs) + goto out; + } while (test_and_set_bit_lock(tag, &hba->lrb_in_use)); + + *tag_out = tag; + ret = true; +out: + return ret; +} + +static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag) +{ + clear_bit_unlock(tag, &hba->lrb_in_use); +} + +/** + * ufshcd_exec_dev_cmd - API for sending device management requests + * @hba - UFS hba + * @cmd_type - specifies the type (NOP, Query...) + * @timeout - time in seconds + * + * NOTE: Since there is only one available tag for device management commands, + * it is expected you hold the hba->dev_cmd.lock mutex. + */ +static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, + enum dev_cmd_type cmd_type, int timeout) +{ + struct ufshcd_lrb *lrbp; + int err; + int tag; + struct completion wait; + unsigned long flags; + + /* + * Get free slot, sleep if slots are unavailable. + * Even though we use wait_event() which sleeps indefinitely, + * the maximum wait time is bounded by SCSI request timeout. + */ + wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag)); + + init_completion(&wait); + lrbp = &hba->lrb[tag]; + WARN_ON(lrbp->cmd); + err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); + if (unlikely(err)) + goto out_put_tag; + + hba->dev_cmd.complete = &wait; + + spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_send_command(hba, tag); + spin_unlock_irqrestore(hba->host->host_lock, flags); + + err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); + +out_put_tag: + ufshcd_put_dev_cmd_tag(hba, tag); + wake_up(&hba->dev_cmd.tag_wq); + return err; +} + +/** + * ufshcd_query_flag() - API function for sending flag query requests + * hba: per-adapter instance + * query_opcode: flag query to perform + * idn: flag idn to access + * flag_res: the flag value after the query request completes + * + * Returns 0 for success, non-zero in case of failure + */ +static int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, + enum flag_idn idn, bool *flag_res) +{ + struct ufs_query_req *request; + struct ufs_query_res *response; + int err; + + BUG_ON(!hba); + + mutex_lock(&hba->dev_cmd.lock); + request = &hba->dev_cmd.query.request; + response = &hba->dev_cmd.query.response; + memset(request, 0, sizeof(struct ufs_query_req)); + memset(response, 0, sizeof(struct ufs_query_res)); + + switch (opcode) { + case UPIU_QUERY_OPCODE_SET_FLAG: + case UPIU_QUERY_OPCODE_CLEAR_FLAG: + case UPIU_QUERY_OPCODE_TOGGLE_FLAG: + request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; + break; + case UPIU_QUERY_OPCODE_READ_FLAG: + request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; + if (!flag_res) { + /* No dummy reads */ + dev_err(hba->dev, "%s: Invalid argument for read request\n", + __func__); + err = -EINVAL; + goto out_unlock; + } + break; + default: + dev_err(hba->dev, + "%s: Expected query flag opcode but got = %d\n", + __func__, opcode); + err = -EINVAL; + goto out_unlock; + } + request->upiu_req.opcode = opcode; + request->upiu_req.idn = idn; + + /* Send query request */ + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, + QUERY_REQ_TIMEOUT); + + if (err) { + dev_err(hba->dev, + "%s: Sending flag query for idn %d failed, err = %d\n", + __func__, idn, err); + goto out_unlock; + } + + if (flag_res) + *flag_res = (response->upiu_res.value & + MASK_QUERY_UPIU_FLAG_LOC) & 0x1; + +out_unlock: + mutex_unlock(&hba->dev_cmd.lock); + return err; +} + +/** + * ufshcd_query_attr - API function for sending attribute requests + * hba: per-adapter instance + * opcode: attribute opcode + * idn: attribute idn to access + * index: index field + * selector: selector field + * attr_val: the attribute value after the query request completes + * + * Returns 0 for success, non-zero in case of failure +*/ +int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, + enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) +{ + struct ufs_query_req *request; + struct ufs_query_res *response; + int err; + + BUG_ON(!hba); + + if (!attr_val) { + dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", + __func__, opcode); + err = -EINVAL; + goto out; + } + + mutex_lock(&hba->dev_cmd.lock); + request = &hba->dev_cmd.query.request; + response = &hba->dev_cmd.query.response; + memset(request, 0, sizeof(struct ufs_query_req)); + memset(response, 0, sizeof(struct ufs_query_res)); + + switch (opcode) { + case UPIU_QUERY_OPCODE_WRITE_ATTR: + request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; + request->upiu_req.value = *attr_val; + break; + case UPIU_QUERY_OPCODE_READ_ATTR: + request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; + break; + default: + dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n", + __func__, opcode); + err = -EINVAL; + goto out_unlock; + } + + request->upiu_req.opcode = opcode; + request->upiu_req.idn = idn; + request->upiu_req.index = index; + request->upiu_req.selector = selector; + + /* Send query request */ + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, + QUERY_REQ_TIMEOUT); + + if (err) { + dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n", + __func__, opcode, idn, err); + goto out_unlock; + } + + *attr_val = response->upiu_res.value; + +out_unlock: + mutex_unlock(&hba->dev_cmd.lock); +out: + return err; +} + /** * ufshcd_memory_alloc - allocate memory for host memory space data structures * @hba: per adapter instance @@ -774,8 +1327,8 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); hba->lrb[i].utr_descriptor_ptr = (utrdlp + i); - hba->lrb[i].ucd_cmd_ptr = - (struct utp_upiu_cmd *)(cmd_descp + i); + hba->lrb[i].ucd_req_ptr = + (struct utp_upiu_req *)(cmd_descp + i); hba->lrb[i].ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu; hba->lrb[i].ucd_prdt_ptr = @@ -809,6 +1362,57 @@ static int ufshcd_dme_link_startup(struct ufs_hba *hba) } /** + * ufshcd_complete_dev_init() - checks device readiness + * hba: per-adapter instance + * + * Set fDeviceInit flag and poll until device toggles it. + */ +static int ufshcd_complete_dev_init(struct ufs_hba *hba) +{ + int i, retries, err = 0; + bool flag_res = 1; + + for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { + /* Set the fDeviceInit flag */ + err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, + QUERY_FLAG_IDN_FDEVICEINIT, NULL); + if (!err || err == -ETIMEDOUT) + break; + dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); + } + if (err) { + dev_err(hba->dev, + "%s setting fDeviceInit flag failed with error %d\n", + __func__, err); + goto out; + } + + /* poll for max. 100 iterations for fDeviceInit flag to clear */ + for (i = 0; i < 100 && !err && flag_res; i++) { + for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { + err = ufshcd_query_flag(hba, + UPIU_QUERY_OPCODE_READ_FLAG, + QUERY_FLAG_IDN_FDEVICEINIT, &flag_res); + if (!err || err == -ETIMEDOUT) + break; + dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, + err); + } + } + if (err) + dev_err(hba->dev, + "%s reading fDeviceInit flag failed with error %d\n", + __func__, err); + else if (flag_res) + dev_err(hba->dev, + "%s fDeviceInit was not cleared by the device\n", + __func__); + +out: + return err; +} + +/** * ufshcd_make_hba_operational - Make UFS controller operational * @hba: per adapter instance * @@ -961,6 +1565,38 @@ out: } /** + * ufshcd_verify_dev_init() - Verify device initialization + * @hba: per-adapter instance + * + * Send NOP OUT UPIU and wait for NOP IN response to check whether the + * device Transport Protocol (UTP) layer is ready after a reset. + * If the UTP layer at the device side is not initialized, it may + * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT + * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. + */ +static int ufshcd_verify_dev_init(struct ufs_hba *hba) +{ + int err = 0; + int retries; + + mutex_lock(&hba->dev_cmd.lock); + for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, + NOP_OUT_TIMEOUT); + + if (!err || err == -ETIMEDOUT) + break; + + dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); + } + mutex_unlock(&hba->dev_cmd.lock); + + if (err) + dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); + return err; +} + +/** * ufshcd_do_reset - reset the host controller * @hba: per adapter instance * @@ -986,13 +1622,20 @@ static int ufshcd_do_reset(struct ufs_hba *hba) for (tag = 0; tag < hba->nutrs; tag++) { if (test_bit(tag, &hba->outstanding_reqs)) { lrbp = &hba->lrb[tag]; - scsi_dma_unmap(lrbp->cmd); - lrbp->cmd->result = DID_RESET << 16; - lrbp->cmd->scsi_done(lrbp->cmd); - lrbp->cmd = NULL; + if (lrbp->cmd) { + scsi_dma_unmap(lrbp->cmd); + lrbp->cmd->result = DID_RESET << 16; + lrbp->cmd->scsi_done(lrbp->cmd); + lrbp->cmd = NULL; + clear_bit_unlock(tag, &hba->lrb_in_use); + } } } + /* complete device management command */ + if (hba->dev_cmd.complete) + complete(hba->dev_cmd.complete); + /* clear outstanding request/task bit maps */ hba->outstanding_reqs = 0; hba->outstanding_tasks = 0; @@ -1199,27 +1842,39 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) switch (ocs) { case OCS_SUCCESS: + result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); - /* check if the returned transfer response is valid */ - result = ufshcd_is_valid_req_rsp(lrbp->ucd_rsp_ptr); - if (result) { + switch (result) { + case UPIU_TRANSACTION_RESPONSE: + /* + * get the response UPIU result to extract + * the SCSI command status + */ + result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); + + /* + * get the result based on SCSI status response + * to notify the SCSI midlayer of the command status + */ + scsi_status = result & MASK_SCSI_STATUS; + result = ufshcd_scsi_cmd_status(lrbp, scsi_status); + + if (ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) + schedule_work(&hba->eeh_work); + break; + case UPIU_TRANSACTION_REJECT_UPIU: + /* TODO: handle Reject UPIU Response */ + result = DID_ERROR << 16; + dev_err(hba->dev, + "Reject UPIU not fully implemented\n"); + break; + default: + result = DID_ERROR << 16; dev_err(hba->dev, - "Invalid response = %x\n", result); + "Unexpected request response code = %x\n", + result); break; } - - /* - * get the response UPIU result to extract - * the SCSI command status - */ - result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); - - /* - * get the result based on SCSI status response - * to notify the SCSI midlayer of the command status - */ - scsi_status = result & MASK_SCSI_STATUS; - result = ufshcd_scsi_cmd_status(lrbp, scsi_status); break; case OCS_ABORTED: result |= DID_ABORT << 16; @@ -1259,28 +1914,40 @@ static void ufshcd_uic_cmd_compl(struct ufs_hba *hba) */ static void ufshcd_transfer_req_compl(struct ufs_hba *hba) { - struct ufshcd_lrb *lrb; + struct ufshcd_lrb *lrbp; + struct scsi_cmnd *cmd; unsigned long completed_reqs; u32 tr_doorbell; int result; int index; + bool int_aggr_reset = false; - lrb = hba->lrb; tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); completed_reqs = tr_doorbell ^ hba->outstanding_reqs; for (index = 0; index < hba->nutrs; index++) { if (test_bit(index, &completed_reqs)) { + lrbp = &hba->lrb[index]; + cmd = lrbp->cmd; + /* + * Don't skip resetting interrupt aggregation counters + * if a regular command is present. + */ + int_aggr_reset |= !lrbp->intr_cmd; - result = ufshcd_transfer_rsp_status(hba, &lrb[index]); - - if (lrb[index].cmd) { - scsi_dma_unmap(lrb[index].cmd); - lrb[index].cmd->result = result; - lrb[index].cmd->scsi_done(lrb[index].cmd); - + if (cmd) { + result = ufshcd_transfer_rsp_status(hba, lrbp); + scsi_dma_unmap(cmd); + cmd->result = result; /* Mark completed command as NULL in LRB */ - lrb[index].cmd = NULL; + lrbp->cmd = NULL; + clear_bit_unlock(index, &hba->lrb_in_use); + /* Do not touch lrbp after scsi done */ + cmd->scsi_done(cmd); + } else if (lrbp->command_type == + UTP_CMD_TYPE_DEV_MANAGE) { + if (hba->dev_cmd.complete) + complete(hba->dev_cmd.complete); } } /* end of if */ } /* end of for */ @@ -1288,8 +1955,238 @@ static void ufshcd_transfer_req_compl(struct ufs_hba *hba) /* clear corresponding bits of completed commands */ hba->outstanding_reqs ^= completed_reqs; + /* we might have free'd some tags above */ + wake_up(&hba->dev_cmd.tag_wq); + /* Reset interrupt aggregation counters */ - ufshcd_config_int_aggr(hba, INT_AGGR_RESET); + if (int_aggr_reset) + ufshcd_config_int_aggr(hba, INT_AGGR_RESET); +} + +/** + * ufshcd_disable_ee - disable exception event + * @hba: per-adapter instance + * @mask: exception event to disable + * + * Disables exception event in the device so that the EVENT_ALERT + * bit is not set. + * + * Returns zero on success, non-zero error value on failure. + */ +static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) +{ + int err = 0; + u32 val; + + if (!(hba->ee_ctrl_mask & mask)) + goto out; + + val = hba->ee_ctrl_mask & ~mask; + val &= 0xFFFF; /* 2 bytes */ + err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val); + if (!err) + hba->ee_ctrl_mask &= ~mask; +out: + return err; +} + +/** + * ufshcd_enable_ee - enable exception event + * @hba: per-adapter instance + * @mask: exception event to enable + * + * Enable corresponding exception event in the device to allow + * device to alert host in critical scenarios. + * + * Returns zero on success, non-zero error value on failure. + */ +static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) +{ + int err = 0; + u32 val; + + if (hba->ee_ctrl_mask & mask) + goto out; + + val = hba->ee_ctrl_mask | mask; + val &= 0xFFFF; /* 2 bytes */ + err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val); + if (!err) + hba->ee_ctrl_mask |= mask; +out: + return err; +} + +/** + * ufshcd_enable_auto_bkops - Allow device managed BKOPS + * @hba: per-adapter instance + * + * Allow device to manage background operations on its own. Enabling + * this might lead to inconsistent latencies during normal data transfers + * as the device is allowed to manage its own way of handling background + * operations. + * + * Returns zero on success, non-zero on failure. + */ +static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) +{ + int err = 0; + + if (hba->auto_bkops_enabled) + goto out; + + err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, + QUERY_FLAG_IDN_BKOPS_EN, NULL); + if (err) { + dev_err(hba->dev, "%s: failed to enable bkops %d\n", + __func__, err); + goto out; + } + + hba->auto_bkops_enabled = true; + + /* No need of URGENT_BKOPS exception from the device */ + err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); + if (err) + dev_err(hba->dev, "%s: failed to disable exception event %d\n", + __func__, err); +out: + return err; +} + +/** + * ufshcd_disable_auto_bkops - block device in doing background operations + * @hba: per-adapter instance + * + * Disabling background operations improves command response latency but + * has drawback of device moving into critical state where the device is + * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the + * host is idle so that BKOPS are managed effectively without any negative + * impacts. + * + * Returns zero on success, non-zero on failure. + */ +static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) +{ + int err = 0; + + if (!hba->auto_bkops_enabled) + goto out; + + /* + * If host assisted BKOPs is to be enabled, make sure + * urgent bkops exception is allowed. + */ + err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS); + if (err) { + dev_err(hba->dev, "%s: failed to enable exception event %d\n", + __func__, err); + goto out; + } + + err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, + QUERY_FLAG_IDN_BKOPS_EN, NULL); + if (err) { + dev_err(hba->dev, "%s: failed to disable bkops %d\n", + __func__, err); + ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); + goto out; + } + + hba->auto_bkops_enabled = false; +out: + return err; +} + +/** + * ufshcd_force_reset_auto_bkops - force enable of auto bkops + * @hba: per adapter instance + * + * After a device reset the device may toggle the BKOPS_EN flag + * to default value. The s/w tracking variables should be updated + * as well. Do this by forcing enable of auto bkops. + */ +static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) +{ + hba->auto_bkops_enabled = false; + hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; + ufshcd_enable_auto_bkops(hba); +} + +static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) +{ + return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status); +} + +/** + * ufshcd_urgent_bkops - handle urgent bkops exception event + * @hba: per-adapter instance + * + * Enable fBackgroundOpsEn flag in the device to permit background + * operations. + */ +static int ufshcd_urgent_bkops(struct ufs_hba *hba) +{ + int err; + u32 status = 0; + + err = ufshcd_get_bkops_status(hba, &status); + if (err) { + dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", + __func__, err); + goto out; + } + + status = status & 0xF; + + /* handle only if status indicates performance impact or critical */ + if (status >= BKOPS_STATUS_PERF_IMPACT) + err = ufshcd_enable_auto_bkops(hba); +out: + return err; +} + +static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) +{ + return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_EE_STATUS, 0, 0, status); +} + +/** + * ufshcd_exception_event_handler - handle exceptions raised by device + * @work: pointer to work data + * + * Read bExceptionEventStatus attribute from the device and handle the + * exception event accordingly. + */ +static void ufshcd_exception_event_handler(struct work_struct *work) +{ + struct ufs_hba *hba; + int err; + u32 status = 0; + hba = container_of(work, struct ufs_hba, eeh_work); + + pm_runtime_get_sync(hba->dev); + err = ufshcd_get_ee_status(hba, &status); + if (err) { + dev_err(hba->dev, "%s: failed to get exception status %d\n", + __func__, err); + goto out; + } + + status &= hba->ee_ctrl_mask; + if (status & MASK_EE_URGENT_BKOPS) { + err = ufshcd_urgent_bkops(hba); + if (err) + dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n", + __func__, err); + } +out: + pm_runtime_put_sync(hba->dev); + return; } /** @@ -1301,9 +2198,11 @@ static void ufshcd_fatal_err_handler(struct work_struct *work) struct ufs_hba *hba; hba = container_of(work, struct ufs_hba, feh_workq); + pm_runtime_get_sync(hba->dev); /* check if reset is already in progress */ if (hba->ufshcd_state != UFSHCD_STATE_RESET) ufshcd_do_reset(hba); + pm_runtime_put_sync(hba->dev); } /** @@ -1432,10 +2331,10 @@ ufshcd_issue_tm_cmd(struct ufs_hba *hba, task_req_upiup = (struct utp_upiu_task_req *) task_req_descp->task_req_upiu; task_req_upiup->header.dword_0 = - cpu_to_be32(UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0, - lrbp->lun, lrbp->task_tag)); + UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0, + lrbp->lun, lrbp->task_tag); task_req_upiup->header.dword_1 = - cpu_to_be32(UPIU_HEADER_DWORD(0, tm_function, 0, 0)); + UPIU_HEADER_DWORD(0, tm_function, 0, 0); task_req_upiup->input_param1 = lrbp->lun; task_req_upiup->input_param1 = @@ -1502,9 +2401,11 @@ static int ufshcd_device_reset(struct scsi_cmnd *cmd) if (hba->lrb[pos].cmd) { scsi_dma_unmap(hba->lrb[pos].cmd); hba->lrb[pos].cmd->result = - DID_ABORT << 16; + DID_ABORT << 16; hba->lrb[pos].cmd->scsi_done(cmd); hba->lrb[pos].cmd = NULL; + clear_bit_unlock(pos, &hba->lrb_in_use); + wake_up(&hba->dev_cmd.tag_wq); } } } /* end of for */ @@ -1572,6 +2473,9 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) __clear_bit(tag, &hba->outstanding_reqs); hba->lrb[tag].cmd = NULL; spin_unlock_irqrestore(host->host_lock, flags); + + clear_bit_unlock(tag, &hba->lrb_in_use); + wake_up(&hba->dev_cmd.tag_wq); out: return err; } @@ -1587,8 +2491,22 @@ static void ufshcd_async_scan(void *data, async_cookie_t cookie) int ret; ret = ufshcd_link_startup(hba); - if (!ret) - scsi_scan_host(hba->host); + if (ret) + goto out; + + ret = ufshcd_verify_dev_init(hba); + if (ret) + goto out; + + ret = ufshcd_complete_dev_init(hba); + if (ret) + goto out; + + ufshcd_force_reset_auto_bkops(hba); + scsi_scan_host(hba->host); + pm_runtime_put_sync(hba->dev); +out: + return; } static struct scsi_host_template ufshcd_driver_template = { @@ -1650,6 +2568,34 @@ int ufshcd_resume(struct ufs_hba *hba) } EXPORT_SYMBOL_GPL(ufshcd_resume); +int ufshcd_runtime_suspend(struct ufs_hba *hba) +{ + if (!hba) + return 0; + + /* + * The device is idle with no requests in the queue, + * allow background operations. + */ + return ufshcd_enable_auto_bkops(hba); +} +EXPORT_SYMBOL(ufshcd_runtime_suspend); + +int ufshcd_runtime_resume(struct ufs_hba *hba) +{ + if (!hba) + return 0; + + return ufshcd_disable_auto_bkops(hba); +} +EXPORT_SYMBOL(ufshcd_runtime_resume); + +int ufshcd_runtime_idle(struct ufs_hba *hba) +{ + return 0; +} +EXPORT_SYMBOL(ufshcd_runtime_idle); + /** * ufshcd_remove - de-allocate SCSI host and host memory space * data structure memory @@ -1657,11 +2603,11 @@ EXPORT_SYMBOL_GPL(ufshcd_resume); */ void ufshcd_remove(struct ufs_hba *hba) { + scsi_remove_host(hba->host); /* disable interrupts */ ufshcd_disable_intr(hba, hba->intr_mask); ufshcd_hba_stop(hba); - scsi_remove_host(hba->host); scsi_host_put(hba->host); } EXPORT_SYMBOL_GPL(ufshcd_remove); @@ -1740,10 +2686,17 @@ int ufshcd_init(struct device *dev, struct ufs_hba **hba_handle, /* Initialize work queues */ INIT_WORK(&hba->feh_workq, ufshcd_fatal_err_handler); + INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); /* Initialize UIC command mutex */ mutex_init(&hba->uic_cmd_mutex); + /* Initialize mutex for device management commands */ + mutex_init(&hba->dev_cmd.lock); + + /* Initialize device management tag acquire wait queue */ + init_waitqueue_head(&hba->dev_cmd.tag_wq); + /* IRQ registration */ err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); if (err) { @@ -1773,6 +2726,9 @@ int ufshcd_init(struct device *dev, struct ufs_hba **hba_handle, *hba_handle = hba; + /* Hold auto suspend until async scan completes */ + pm_runtime_get_sync(dev); + async_schedule(ufshcd_async_scan, hba); return 0; diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index 49590ee07acc..59c9c4848be1 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -68,6 +68,11 @@ #define UFSHCD "ufshcd" #define UFSHCD_DRIVER_VERSION "0.2" +enum dev_cmd_type { + DEV_CMD_TYPE_NOP = 0x0, + DEV_CMD_TYPE_QUERY = 0x1, +}; + /** * struct uic_command - UIC command structure * @command: UIC command @@ -91,7 +96,7 @@ struct uic_command { /** * struct ufshcd_lrb - local reference block * @utr_descriptor_ptr: UTRD address of the command - * @ucd_cmd_ptr: UCD address of the command + * @ucd_req_ptr: UCD address of the command * @ucd_rsp_ptr: Response UPIU address for this command * @ucd_prdt_ptr: PRDT address of the command * @cmd: pointer to SCSI command @@ -101,10 +106,11 @@ struct uic_command { * @command_type: SCSI, UFS, Query. * @task_tag: Task tag of the command * @lun: LUN of the command + * @intr_cmd: Interrupt command (doesn't participate in interrupt aggregation) */ struct ufshcd_lrb { struct utp_transfer_req_desc *utr_descriptor_ptr; - struct utp_upiu_cmd *ucd_cmd_ptr; + struct utp_upiu_req *ucd_req_ptr; struct utp_upiu_rsp *ucd_rsp_ptr; struct ufshcd_sg_entry *ucd_prdt_ptr; @@ -116,8 +122,35 @@ struct ufshcd_lrb { int command_type; int task_tag; unsigned int lun; + bool intr_cmd; }; +/** + * struct ufs_query - holds relevent data structures for query request + * @request: request upiu and function + * @descriptor: buffer for sending/receiving descriptor + * @response: response upiu and response + */ +struct ufs_query { + struct ufs_query_req request; + u8 *descriptor; + struct ufs_query_res response; +}; + +/** + * struct ufs_dev_cmd - all assosiated fields with device management commands + * @type: device management command type - Query, NOP OUT + * @lock: lock to allow one command at a time + * @complete: internal commands completion + * @tag_wq: wait queue until free command slot is available + */ +struct ufs_dev_cmd { + enum dev_cmd_type type; + struct mutex lock; + struct completion *complete; + wait_queue_head_t tag_wq; + struct ufs_query query; +}; /** * struct ufs_hba - per adapter private structure @@ -131,6 +164,7 @@ struct ufshcd_lrb { * @host: Scsi_Host instance of the driver * @dev: device handle * @lrb: local reference block + * @lrb_in_use: lrb in use * @outstanding_tasks: Bits representing outstanding task requests * @outstanding_reqs: Bits representing outstanding transfer requests * @capabilities: UFS Controller Capabilities @@ -144,8 +178,12 @@ struct ufshcd_lrb { * @tm_condition: condition variable for task management * @ufshcd_state: UFSHCD states * @intr_mask: Interrupt Mask Bits + * @ee_ctrl_mask: Exception event control mask * @feh_workq: Work queue for fatal controller error handling + * @eeh_work: Worker to handle exception events * @errors: HBA errors + * @dev_cmd: ufs device management command information + * @auto_bkops_enabled: to track whether bkops is enabled in device */ struct ufs_hba { void __iomem *mmio_base; @@ -164,6 +202,7 @@ struct ufs_hba { struct device *dev; struct ufshcd_lrb *lrb; + unsigned long lrb_in_use; unsigned long outstanding_tasks; unsigned long outstanding_reqs; @@ -182,12 +221,19 @@ struct ufs_hba { u32 ufshcd_state; u32 intr_mask; + u16 ee_ctrl_mask; /* Work Queues */ struct work_struct feh_workq; + struct work_struct eeh_work; /* HBA Errors */ u32 errors; + + /* Device management request data */ + struct ufs_dev_cmd dev_cmd; + + bool auto_bkops_enabled; }; #define ufshcd_writel(hba, val, reg) \ @@ -208,4 +254,13 @@ static inline void ufshcd_hba_stop(struct ufs_hba *hba) ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); } +static inline void check_upiu_size(void) +{ + BUILD_BUG_ON(ALIGNED_UPIU_SIZE < + GENERAL_UPIU_REQUEST_SIZE + QUERY_DESC_MAX_SIZE); +} + +extern int ufshcd_runtime_suspend(struct ufs_hba *hba); +extern int ufshcd_runtime_resume(struct ufs_hba *hba); +extern int ufshcd_runtime_idle(struct ufs_hba *hba); #endif /* End of Header */ diff --git a/drivers/scsi/ufs/ufshci.h b/drivers/scsi/ufs/ufshci.h index d5c5f1482d7d..f1e1b7459107 100644 --- a/drivers/scsi/ufs/ufshci.h +++ b/drivers/scsi/ufs/ufshci.h @@ -39,7 +39,7 @@ enum { TASK_REQ_UPIU_SIZE_DWORDS = 8, TASK_RSP_UPIU_SIZE_DWORDS = 8, - ALIGNED_UPIU_SIZE = 128, + ALIGNED_UPIU_SIZE = 512, }; /* UFSHCI Registers */ |
