xref: /linux/drivers/ufs/core/ufshcd.c (revision cb7e509c)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Universal Flash Storage Host controller driver Core
4  * Copyright (C) 2011-2013 Samsung India Software Operations
5  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
6  *
7  * Authors:
8  *	Santosh Yaraganavi <santosh.sy@samsung.com>
9  *	Vinayak Holikatti <h.vinayak@samsung.com>
10  */
11 
12 #include <linux/async.h>
13 #include <linux/devfreq.h>
14 #include <linux/nls.h>
15 #include <linux/of.h>
16 #include <linux/bitfield.h>
17 #include <linux/blk-pm.h>
18 #include <linux/blkdev.h>
19 #include <linux/clk.h>
20 #include <linux/delay.h>
21 #include <linux/interrupt.h>
22 #include <linux/module.h>
23 #include <linux/pm_opp.h>
24 #include <linux/regulator/consumer.h>
25 #include <linux/sched/clock.h>
26 #include <linux/iopoll.h>
27 #include <scsi/scsi_cmnd.h>
28 #include <scsi/scsi_dbg.h>
29 #include <scsi/scsi_driver.h>
30 #include <scsi/scsi_eh.h>
31 #include "ufshcd-priv.h"
32 #include <ufs/ufs_quirks.h>
33 #include <ufs/unipro.h>
34 #include "ufs-sysfs.h"
35 #include "ufs-debugfs.h"
36 #include "ufs-fault-injection.h"
37 #include "ufs_bsg.h"
38 #include "ufshcd-crypto.h"
39 #include <linux/unaligned.h>
40 
41 #define CREATE_TRACE_POINTS
42 #include "ufs_trace.h"
43 
44 #define UFSHCD_ENABLE_INTRS	(UTP_TRANSFER_REQ_COMPL |\
45 				 UTP_TASK_REQ_COMPL |\
46 				 UFSHCD_ERROR_MASK)
47 
48 #define UFSHCD_ENABLE_MCQ_INTRS	(UTP_TASK_REQ_COMPL |\
49 				 UFSHCD_ERROR_MASK |\
50 				 MCQ_CQ_EVENT_STATUS)
51 
52 
53 /* UIC command timeout, unit: ms */
54 enum {
55 	UIC_CMD_TIMEOUT_DEFAULT	= 500,
56 	UIC_CMD_TIMEOUT_MAX	= 2000,
57 };
58 /* NOP OUT retries waiting for NOP IN response */
59 #define NOP_OUT_RETRIES    10
60 /* Timeout after 50 msecs if NOP OUT hangs without response */
61 #define NOP_OUT_TIMEOUT    50 /* msecs */
62 
63 /* Query request retries */
64 #define QUERY_REQ_RETRIES 3
65 /* Query request timeout */
66 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
67 
68 /* Advanced RPMB request timeout */
69 #define ADVANCED_RPMB_REQ_TIMEOUT  3000 /* 3 seconds */
70 
71 /* Task management command timeout */
72 #define TM_CMD_TIMEOUT	100 /* msecs */
73 
74 /* maximum number of retries for a general UIC command  */
75 #define UFS_UIC_COMMAND_RETRIES 3
76 
77 /* maximum number of link-startup retries */
78 #define DME_LINKSTARTUP_RETRIES 3
79 
80 /* maximum number of reset retries before giving up */
81 #define MAX_HOST_RESET_RETRIES 5
82 
83 /* Maximum number of error handler retries before giving up */
84 #define MAX_ERR_HANDLER_RETRIES 5
85 
86 /* Expose the flag value from utp_upiu_query.value */
87 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
88 
89 /* Interrupt aggregation default timeout, unit: 40us */
90 #define INT_AGGR_DEF_TO	0x02
91 
92 /* default delay of autosuspend: 2000 ms */
93 #define RPM_AUTOSUSPEND_DELAY_MS 2000
94 
95 /* Default delay of RPM device flush delayed work */
96 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
97 
98 /* Default value of wait time before gating device ref clock */
99 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
100 
101 /* Polling time to wait for fDeviceInit */
102 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
103 
104 /* Default RTC update every 10 seconds */
105 #define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC)
106 
107 /* bMaxNumOfRTT is equal to two after device manufacturing */
108 #define DEFAULT_MAX_NUM_RTT 2
109 
110 /* UFSHC 4.0 compliant HC support this mode. */
111 static bool use_mcq_mode = true;
112 
is_mcq_supported(struct ufs_hba * hba)113 static bool is_mcq_supported(struct ufs_hba *hba)
114 {
115 	return hba->mcq_sup && use_mcq_mode;
116 }
117 
118 module_param(use_mcq_mode, bool, 0644);
119 MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default");
120 
121 static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT;
122 
uic_cmd_timeout_set(const char * val,const struct kernel_param * kp)123 static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp)
124 {
125 	return param_set_uint_minmax(val, kp, UIC_CMD_TIMEOUT_DEFAULT,
126 				     UIC_CMD_TIMEOUT_MAX);
127 }
128 
129 static const struct kernel_param_ops uic_cmd_timeout_ops = {
130 	.set = uic_cmd_timeout_set,
131 	.get = param_get_uint,
132 };
133 
134 module_param_cb(uic_cmd_timeout, &uic_cmd_timeout_ops, &uic_cmd_timeout, 0644);
135 MODULE_PARM_DESC(uic_cmd_timeout,
136 		 "UFS UIC command timeout in milliseconds. Defaults to 500ms. Supported values range from 500ms to 2 seconds inclusively");
137 
138 #define ufshcd_toggle_vreg(_dev, _vreg, _on)				\
139 	({                                                              \
140 		int _ret;                                               \
141 		if (_on)                                                \
142 			_ret = ufshcd_enable_vreg(_dev, _vreg);         \
143 		else                                                    \
144 			_ret = ufshcd_disable_vreg(_dev, _vreg);        \
145 		_ret;                                                   \
146 	})
147 
148 #define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
149 	size_t __len = (len);                                            \
150 	print_hex_dump(KERN_ERR, prefix_str,                             \
151 		       __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
152 		       16, 4, buf, __len, false);                        \
153 } while (0)
154 
ufshcd_dump_regs(struct ufs_hba * hba,size_t offset,size_t len,const char * prefix)155 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
156 		     const char *prefix)
157 {
158 	u32 *regs;
159 	size_t pos;
160 
161 	if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
162 		return -EINVAL;
163 
164 	regs = kzalloc(len, GFP_ATOMIC);
165 	if (!regs)
166 		return -ENOMEM;
167 
168 	for (pos = 0; pos < len; pos += 4) {
169 		if (offset == 0 &&
170 		    pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER &&
171 		    pos <= REG_UIC_ERROR_CODE_DME)
172 			continue;
173 		regs[pos / 4] = ufshcd_readl(hba, offset + pos);
174 	}
175 
176 	ufshcd_hex_dump(prefix, regs, len);
177 	kfree(regs);
178 
179 	return 0;
180 }
181 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
182 
183 enum {
184 	UFSHCD_MAX_CHANNEL	= 0,
185 	UFSHCD_MAX_ID		= 1,
186 };
187 
188 static const char *const ufshcd_state_name[] = {
189 	[UFSHCD_STATE_RESET]			= "reset",
190 	[UFSHCD_STATE_OPERATIONAL]		= "operational",
191 	[UFSHCD_STATE_ERROR]			= "error",
192 	[UFSHCD_STATE_EH_SCHEDULED_FATAL]	= "eh_fatal",
193 	[UFSHCD_STATE_EH_SCHEDULED_NON_FATAL]	= "eh_non_fatal",
194 };
195 
196 /* UFSHCD error handling flags */
197 enum {
198 	UFSHCD_EH_IN_PROGRESS = (1 << 0),
199 };
200 
201 /* UFSHCD UIC layer error flags */
202 enum {
203 	UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
204 	UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
205 	UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
206 	UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
207 	UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
208 	UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
209 	UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
210 };
211 
212 #define ufshcd_set_eh_in_progress(h) \
213 	((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
214 #define ufshcd_eh_in_progress(h) \
215 	((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
216 #define ufshcd_clear_eh_in_progress(h) \
217 	((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
218 
219 const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
220 	[UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
221 	[UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
222 	[UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
223 	[UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
224 	[UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
225 	[UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
226 	/*
227 	 * For DeepSleep, the link is first put in hibern8 and then off.
228 	 * Leaving the link in hibern8 is not supported.
229 	 */
230 	[UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
231 };
232 
233 static inline enum ufs_dev_pwr_mode
ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)234 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
235 {
236 	return ufs_pm_lvl_states[lvl].dev_state;
237 }
238 
239 static inline enum uic_link_state
ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)240 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
241 {
242 	return ufs_pm_lvl_states[lvl].link_state;
243 }
244 
245 static inline enum ufs_pm_level
ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,enum uic_link_state link_state)246 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
247 					enum uic_link_state link_state)
248 {
249 	enum ufs_pm_level lvl;
250 
251 	for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
252 		if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
253 			(ufs_pm_lvl_states[lvl].link_state == link_state))
254 			return lvl;
255 	}
256 
257 	/* if no match found, return the level 0 */
258 	return UFS_PM_LVL_0;
259 }
260 
ufshcd_is_ufs_dev_busy(struct ufs_hba * hba)261 static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba)
262 {
263 	return (hba->clk_gating.active_reqs || hba->outstanding_reqs || hba->outstanding_tasks ||
264 		hba->active_uic_cmd || hba->uic_async_done);
265 }
266 
267 static const struct ufs_dev_quirk ufs_fixups[] = {
268 	/* UFS cards deviations table */
269 	{ .wmanufacturerid = UFS_VENDOR_MICRON,
270 	  .model = UFS_ANY_MODEL,
271 	  .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
272 	{ .wmanufacturerid = UFS_VENDOR_SAMSUNG,
273 	  .model = UFS_ANY_MODEL,
274 	  .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
275 		   UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
276 		   UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS },
277 	{ .wmanufacturerid = UFS_VENDOR_SKHYNIX,
278 	  .model = UFS_ANY_MODEL,
279 	  .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME },
280 	{ .wmanufacturerid = UFS_VENDOR_SKHYNIX,
281 	  .model = "hB8aL1" /*H28U62301AMR*/,
282 	  .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME },
283 	{ .wmanufacturerid = UFS_VENDOR_TOSHIBA,
284 	  .model = UFS_ANY_MODEL,
285 	  .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
286 	{ .wmanufacturerid = UFS_VENDOR_TOSHIBA,
287 	  .model = "THGLF2G9C8KBADG",
288 	  .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
289 	{ .wmanufacturerid = UFS_VENDOR_TOSHIBA,
290 	  .model = "THGLF2G9D8KBADG",
291 	  .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
292 	{}
293 };
294 
295 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
296 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
297 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
298 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
299 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
300 static void ufshcd_hba_exit(struct ufs_hba *hba);
301 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params);
302 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
303 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
304 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
305 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
306 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
307 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq,
308 			     bool scale_up);
309 static irqreturn_t ufshcd_intr(int irq, void *__hba);
310 static int ufshcd_change_power_mode(struct ufs_hba *hba,
311 			     struct ufs_pa_layer_attr *pwr_mode);
312 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
313 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
314 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
315 					 struct ufs_vreg *vreg);
316 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
317 						 bool enable);
318 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
319 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
320 
ufshcd_enable_irq(struct ufs_hba * hba)321 void ufshcd_enable_irq(struct ufs_hba *hba)
322 {
323 	if (!hba->is_irq_enabled) {
324 		enable_irq(hba->irq);
325 		hba->is_irq_enabled = true;
326 	}
327 }
328 EXPORT_SYMBOL_GPL(ufshcd_enable_irq);
329 
ufshcd_disable_irq(struct ufs_hba * hba)330 void ufshcd_disable_irq(struct ufs_hba *hba)
331 {
332 	if (hba->is_irq_enabled) {
333 		disable_irq(hba->irq);
334 		hba->is_irq_enabled = false;
335 	}
336 }
337 EXPORT_SYMBOL_GPL(ufshcd_disable_irq);
338 
ufshcd_configure_wb(struct ufs_hba * hba)339 static void ufshcd_configure_wb(struct ufs_hba *hba)
340 {
341 	if (!ufshcd_is_wb_allowed(hba))
342 		return;
343 
344 	ufshcd_wb_toggle(hba, true);
345 
346 	ufshcd_wb_toggle_buf_flush_during_h8(hba, true);
347 
348 	if (ufshcd_is_wb_buf_flush_allowed(hba))
349 		ufshcd_wb_toggle_buf_flush(hba, true);
350 }
351 
ufshcd_scsi_unblock_requests(struct ufs_hba * hba)352 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
353 {
354 	if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
355 		scsi_unblock_requests(hba->host);
356 }
357 
ufshcd_scsi_block_requests(struct ufs_hba * hba)358 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
359 {
360 	if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
361 		scsi_block_requests(hba->host);
362 }
363 
ufshcd_add_cmd_upiu_trace(struct ufs_hba * hba,unsigned int tag,enum ufs_trace_str_t str_t)364 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
365 				      enum ufs_trace_str_t str_t)
366 {
367 	struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
368 	struct utp_upiu_header *header;
369 
370 	if (!trace_ufshcd_upiu_enabled())
371 		return;
372 
373 	if (str_t == UFS_CMD_SEND)
374 		header = &rq->header;
375 	else
376 		header = &hba->lrb[tag].ucd_rsp_ptr->header;
377 
378 	trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
379 			  UFS_TSF_CDB);
380 }
381 
ufshcd_add_query_upiu_trace(struct ufs_hba * hba,enum ufs_trace_str_t str_t,struct utp_upiu_req * rq_rsp)382 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
383 					enum ufs_trace_str_t str_t,
384 					struct utp_upiu_req *rq_rsp)
385 {
386 	if (!trace_ufshcd_upiu_enabled())
387 		return;
388 
389 	trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
390 			  &rq_rsp->qr, UFS_TSF_OSF);
391 }
392 
ufshcd_add_tm_upiu_trace(struct ufs_hba * hba,unsigned int tag,enum ufs_trace_str_t str_t)393 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
394 				     enum ufs_trace_str_t str_t)
395 {
396 	struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
397 
398 	if (!trace_ufshcd_upiu_enabled())
399 		return;
400 
401 	if (str_t == UFS_TM_SEND)
402 		trace_ufshcd_upiu(dev_name(hba->dev), str_t,
403 				  &descp->upiu_req.req_header,
404 				  &descp->upiu_req.input_param1,
405 				  UFS_TSF_TM_INPUT);
406 	else
407 		trace_ufshcd_upiu(dev_name(hba->dev), str_t,
408 				  &descp->upiu_rsp.rsp_header,
409 				  &descp->upiu_rsp.output_param1,
410 				  UFS_TSF_TM_OUTPUT);
411 }
412 
ufshcd_add_uic_command_trace(struct ufs_hba * hba,const struct uic_command * ucmd,enum ufs_trace_str_t str_t)413 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
414 					 const struct uic_command *ucmd,
415 					 enum ufs_trace_str_t str_t)
416 {
417 	u32 cmd;
418 
419 	if (!trace_ufshcd_uic_command_enabled())
420 		return;
421 
422 	if (str_t == UFS_CMD_SEND)
423 		cmd = ucmd->command;
424 	else
425 		cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
426 
427 	trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
428 				 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
429 				 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
430 				 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
431 }
432 
ufshcd_add_command_trace(struct ufs_hba * hba,unsigned int tag,enum ufs_trace_str_t str_t)433 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
434 				     enum ufs_trace_str_t str_t)
435 {
436 	u64 lba = 0;
437 	u8 opcode = 0, group_id = 0;
438 	u32 doorbell = 0;
439 	u32 intr;
440 	int hwq_id = -1;
441 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
442 	struct scsi_cmnd *cmd = lrbp->cmd;
443 	struct request *rq = scsi_cmd_to_rq(cmd);
444 	int transfer_len = -1;
445 
446 	if (!cmd)
447 		return;
448 
449 	/* trace UPIU also */
450 	ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
451 	if (!trace_ufshcd_command_enabled())
452 		return;
453 
454 	opcode = cmd->cmnd[0];
455 
456 	if (opcode == READ_10 || opcode == WRITE_10) {
457 		/*
458 		 * Currently we only fully trace read(10) and write(10) commands
459 		 */
460 		transfer_len =
461 		       be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
462 		lba = scsi_get_lba(cmd);
463 		if (opcode == WRITE_10)
464 			group_id = lrbp->cmd->cmnd[6];
465 	} else if (opcode == UNMAP) {
466 		/*
467 		 * The number of Bytes to be unmapped beginning with the lba.
468 		 */
469 		transfer_len = blk_rq_bytes(rq);
470 		lba = scsi_get_lba(cmd);
471 	}
472 
473 	intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
474 
475 	if (hba->mcq_enabled) {
476 		struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
477 
478 		hwq_id = hwq->id;
479 	} else {
480 		doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
481 	}
482 	trace_ufshcd_command(cmd->device, str_t, tag, doorbell, hwq_id,
483 			     transfer_len, intr, lba, opcode, group_id);
484 }
485 
ufshcd_print_clk_freqs(struct ufs_hba * hba)486 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
487 {
488 	struct ufs_clk_info *clki;
489 	struct list_head *head = &hba->clk_list_head;
490 
491 	if (list_empty(head))
492 		return;
493 
494 	list_for_each_entry(clki, head, list) {
495 		if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
496 				clki->max_freq)
497 			dev_err(hba->dev, "clk: %s, rate: %u\n",
498 					clki->name, clki->curr_freq);
499 	}
500 }
501 
ufshcd_print_evt(struct ufs_hba * hba,u32 id,const char * err_name)502 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
503 			     const char *err_name)
504 {
505 	int i;
506 	bool found = false;
507 	const struct ufs_event_hist *e;
508 
509 	if (id >= UFS_EVT_CNT)
510 		return;
511 
512 	e = &hba->ufs_stats.event[id];
513 
514 	for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
515 		int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
516 
517 		if (e->tstamp[p] == 0)
518 			continue;
519 		dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
520 			e->val[p], div_u64(e->tstamp[p], 1000));
521 		found = true;
522 	}
523 
524 	if (!found)
525 		dev_err(hba->dev, "No record of %s\n", err_name);
526 	else
527 		dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
528 }
529 
ufshcd_print_evt_hist(struct ufs_hba * hba)530 static void ufshcd_print_evt_hist(struct ufs_hba *hba)
531 {
532 	ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
533 
534 	ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
535 	ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
536 	ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
537 	ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
538 	ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
539 	ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
540 			 "auto_hibern8_err");
541 	ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
542 	ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
543 			 "link_startup_fail");
544 	ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
545 	ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
546 			 "suspend_fail");
547 	ufshcd_print_evt(hba, UFS_EVT_WL_RES_ERR, "wlun resume_fail");
548 	ufshcd_print_evt(hba, UFS_EVT_WL_SUSP_ERR,
549 			 "wlun suspend_fail");
550 	ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
551 	ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
552 	ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
553 
554 	ufshcd_vops_dbg_register_dump(hba);
555 }
556 
557 static
ufshcd_print_tr(struct ufs_hba * hba,int tag,bool pr_prdt)558 void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
559 {
560 	const struct ufshcd_lrb *lrbp;
561 	int prdt_length;
562 
563 	lrbp = &hba->lrb[tag];
564 
565 	dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
566 			tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
567 	dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
568 			tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000));
569 	dev_err(hba->dev,
570 		"UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
571 		tag, (u64)lrbp->utrd_dma_addr);
572 
573 	ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
574 			sizeof(struct utp_transfer_req_desc));
575 	dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
576 		(u64)lrbp->ucd_req_dma_addr);
577 	ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
578 			sizeof(struct utp_upiu_req));
579 	dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
580 		(u64)lrbp->ucd_rsp_dma_addr);
581 	ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
582 			sizeof(struct utp_upiu_rsp));
583 
584 	prdt_length = le16_to_cpu(
585 		lrbp->utr_descriptor_ptr->prd_table_length);
586 	if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
587 		prdt_length /= ufshcd_sg_entry_size(hba);
588 
589 	dev_err(hba->dev,
590 		"UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
591 		tag, prdt_length,
592 		(u64)lrbp->ucd_prdt_dma_addr);
593 
594 	if (pr_prdt)
595 		ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
596 			ufshcd_sg_entry_size(hba) * prdt_length);
597 }
598 
ufshcd_print_tr_iter(struct request * req,void * priv)599 static bool ufshcd_print_tr_iter(struct request *req, void *priv)
600 {
601 	struct scsi_device *sdev = req->q->queuedata;
602 	struct Scsi_Host *shost = sdev->host;
603 	struct ufs_hba *hba = shost_priv(shost);
604 
605 	ufshcd_print_tr(hba, req->tag, *(bool *)priv);
606 
607 	return true;
608 }
609 
610 /**
611  * ufshcd_print_trs_all - print trs for all started requests.
612  * @hba: per-adapter instance.
613  * @pr_prdt: need to print prdt or not.
614  */
ufshcd_print_trs_all(struct ufs_hba * hba,bool pr_prdt)615 static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt)
616 {
617 	blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt);
618 }
619 
ufshcd_print_tmrs(struct ufs_hba * hba,unsigned long bitmap)620 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
621 {
622 	int tag;
623 
624 	for_each_set_bit(tag, &bitmap, hba->nutmrs) {
625 		struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
626 
627 		dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
628 		ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
629 	}
630 }
631 
ufshcd_print_host_state(struct ufs_hba * hba)632 static void ufshcd_print_host_state(struct ufs_hba *hba)
633 {
634 	const struct scsi_device *sdev_ufs = hba->ufs_device_wlun;
635 
636 	dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
637 	dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
638 		hba->outstanding_reqs, hba->outstanding_tasks);
639 	dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
640 		hba->saved_err, hba->saved_uic_err);
641 	dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
642 		hba->curr_dev_pwr_mode, hba->uic_link_state);
643 	dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
644 		hba->pm_op_in_progress, hba->is_sys_suspended);
645 	dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
646 		hba->auto_bkops_enabled, hba->host->host_self_blocked);
647 	dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
648 	dev_err(hba->dev,
649 		"last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
650 		div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000),
651 		hba->ufs_stats.hibern8_exit_cnt);
652 	dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
653 		div_u64(hba->ufs_stats.last_intr_ts, 1000),
654 		hba->ufs_stats.last_intr_status);
655 	dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
656 		hba->eh_flags, hba->req_abort_count);
657 	dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
658 		hba->ufs_version, hba->capabilities, hba->caps);
659 	dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
660 		hba->dev_quirks);
661 	if (sdev_ufs)
662 		dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
663 			sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
664 
665 	ufshcd_print_clk_freqs(hba);
666 }
667 
668 /**
669  * ufshcd_print_pwr_info - print power params as saved in hba
670  * power info
671  * @hba: per-adapter instance
672  */
ufshcd_print_pwr_info(struct ufs_hba * hba)673 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
674 {
675 	static const char * const names[] = {
676 		"INVALID MODE",
677 		"FAST MODE",
678 		"SLOW_MODE",
679 		"INVALID MODE",
680 		"FASTAUTO_MODE",
681 		"SLOWAUTO_MODE",
682 		"INVALID MODE",
683 	};
684 
685 	/*
686 	 * Using dev_dbg to avoid messages during runtime PM to avoid
687 	 * never-ending cycles of messages written back to storage by user space
688 	 * causing runtime resume, causing more messages and so on.
689 	 */
690 	dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
691 		 __func__,
692 		 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
693 		 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
694 		 names[hba->pwr_info.pwr_rx],
695 		 names[hba->pwr_info.pwr_tx],
696 		 hba->pwr_info.hs_rate);
697 }
698 
ufshcd_device_reset(struct ufs_hba * hba)699 static void ufshcd_device_reset(struct ufs_hba *hba)
700 {
701 	int err;
702 
703 	err = ufshcd_vops_device_reset(hba);
704 
705 	if (!err) {
706 		ufshcd_set_ufs_dev_active(hba);
707 		if (ufshcd_is_wb_allowed(hba)) {
708 			hba->dev_info.wb_enabled = false;
709 			hba->dev_info.wb_buf_flush_enabled = false;
710 		}
711 		if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE)
712 			hba->dev_info.rtc_time_baseline = 0;
713 	}
714 	if (err != -EOPNOTSUPP)
715 		ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
716 }
717 
ufshcd_delay_us(unsigned long us,unsigned long tolerance)718 void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
719 {
720 	if (!us)
721 		return;
722 
723 	if (us < 10)
724 		udelay(us);
725 	else
726 		usleep_range(us, us + tolerance);
727 }
728 EXPORT_SYMBOL_GPL(ufshcd_delay_us);
729 
730 /**
731  * ufshcd_wait_for_register - wait for register value to change
732  * @hba: per-adapter interface
733  * @reg: mmio register offset
734  * @mask: mask to apply to the read register value
735  * @val: value to wait for
736  * @interval_us: polling interval in microseconds
737  * @timeout_ms: timeout in milliseconds
738  *
739  * Return: -ETIMEDOUT on error, zero on success.
740  */
ufshcd_wait_for_register(struct ufs_hba * hba,u32 reg,u32 mask,u32 val,unsigned long interval_us,unsigned long timeout_ms)741 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
742 				u32 val, unsigned long interval_us,
743 				unsigned long timeout_ms)
744 {
745 	int err = 0;
746 	unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
747 
748 	/* ignore bits that we don't intend to wait on */
749 	val = val & mask;
750 
751 	while ((ufshcd_readl(hba, reg) & mask) != val) {
752 		usleep_range(interval_us, interval_us + 50);
753 		if (time_after(jiffies, timeout)) {
754 			if ((ufshcd_readl(hba, reg) & mask) != val)
755 				err = -ETIMEDOUT;
756 			break;
757 		}
758 	}
759 
760 	return err;
761 }
762 
763 /**
764  * ufshcd_get_intr_mask - Get the interrupt bit mask
765  * @hba: Pointer to adapter instance
766  *
767  * Return: interrupt bit mask per version
768  */
ufshcd_get_intr_mask(struct ufs_hba * hba)769 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
770 {
771 	if (hba->ufs_version <= ufshci_version(2, 0))
772 		return INTERRUPT_MASK_ALL_VER_11;
773 
774 	return INTERRUPT_MASK_ALL_VER_21;
775 }
776 
777 /**
778  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
779  * @hba: Pointer to adapter instance
780  *
781  * Return: UFSHCI version supported by the controller
782  */
ufshcd_get_ufs_version(struct ufs_hba * hba)783 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
784 {
785 	u32 ufshci_ver;
786 
787 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
788 		ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
789 	else
790 		ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
791 
792 	/*
793 	 * UFSHCI v1.x uses a different version scheme, in order
794 	 * to allow the use of comparisons with the ufshci_version
795 	 * function, we convert it to the same scheme as ufs 2.0+.
796 	 */
797 	if (ufshci_ver & 0x00010000)
798 		return ufshci_version(1, ufshci_ver & 0x00000100);
799 
800 	return ufshci_ver;
801 }
802 
803 /**
804  * ufshcd_is_device_present - Check if any device connected to
805  *			      the host controller
806  * @hba: pointer to adapter instance
807  *
808  * Return: true if device present, false if no device detected
809  */
ufshcd_is_device_present(struct ufs_hba * hba)810 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
811 {
812 	return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT;
813 }
814 
815 /**
816  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
817  * @lrbp: pointer to local command reference block
818  * @cqe: pointer to the completion queue entry
819  *
820  * This function is used to get the OCS field from UTRD
821  *
822  * Return: the OCS field in the UTRD.
823  */
ufshcd_get_tr_ocs(struct ufshcd_lrb * lrbp,struct cq_entry * cqe)824 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp,
825 				      struct cq_entry *cqe)
826 {
827 	if (cqe)
828 		return le32_to_cpu(cqe->status) & MASK_OCS;
829 
830 	return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS;
831 }
832 
833 /**
834  * ufshcd_utrl_clear() - Clear requests from the controller request list.
835  * @hba: per adapter instance
836  * @mask: mask with one bit set for each request to be cleared
837  */
ufshcd_utrl_clear(struct ufs_hba * hba,u32 mask)838 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask)
839 {
840 	if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
841 		mask = ~mask;
842 	/*
843 	 * From the UFSHCI specification: "UTP Transfer Request List CLear
844 	 * Register (UTRLCLR): This field is bit significant. Each bit
845 	 * corresponds to a slot in the UTP Transfer Request List, where bit 0
846 	 * corresponds to request slot 0. A bit in this field is set to ‘0’
847 	 * by host software to indicate to the host controller that a transfer
848 	 * request slot is cleared. The host controller
849 	 * shall free up any resources associated to the request slot
850 	 * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The
851 	 * host software indicates no change to request slots by setting the
852 	 * associated bits in this field to ‘1’. Bits in this field shall only
853 	 * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’."
854 	 */
855 	ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR);
856 }
857 
858 /**
859  * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register
860  * @hba: per adapter instance
861  * @pos: position of the bit to be cleared
862  */
ufshcd_utmrl_clear(struct ufs_hba * hba,u32 pos)863 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
864 {
865 	if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
866 		ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
867 	else
868 		ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
869 }
870 
871 /**
872  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
873  * @reg: Register value of host controller status
874  *
875  * Return: 0 on success; a positive value if failed.
876  */
ufshcd_get_lists_status(u32 reg)877 static inline int ufshcd_get_lists_status(u32 reg)
878 {
879 	return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
880 }
881 
882 /**
883  * ufshcd_get_uic_cmd_result - Get the UIC command result
884  * @hba: Pointer to adapter instance
885  *
886  * This function gets the result of UIC command completion
887  *
888  * Return: 0 on success; non-zero value on error.
889  */
ufshcd_get_uic_cmd_result(struct ufs_hba * hba)890 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
891 {
892 	return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
893 	       MASK_UIC_COMMAND_RESULT;
894 }
895 
896 /**
897  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
898  * @hba: Pointer to adapter instance
899  *
900  * This function gets UIC command argument3
901  *
902  * Return: 0 on success; non-zero value on error.
903  */
ufshcd_get_dme_attr_val(struct ufs_hba * hba)904 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
905 {
906 	return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
907 }
908 
909 /**
910  * ufshcd_get_req_rsp - returns the TR response transaction type
911  * @ucd_rsp_ptr: pointer to response UPIU
912  *
913  * Return: UPIU type.
914  */
915 static inline enum upiu_response_transaction
ufshcd_get_req_rsp(struct utp_upiu_rsp * ucd_rsp_ptr)916 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
917 {
918 	return ucd_rsp_ptr->header.transaction_code;
919 }
920 
921 /**
922  * ufshcd_is_exception_event - Check if the device raised an exception event
923  * @ucd_rsp_ptr: pointer to response UPIU
924  *
925  * The function checks if the device raised an exception event indicated in
926  * the Device Information field of response UPIU.
927  *
928  * Return: true if exception is raised, false otherwise.
929  */
ufshcd_is_exception_event(struct utp_upiu_rsp * ucd_rsp_ptr)930 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
931 {
932 	return ucd_rsp_ptr->header.device_information & 1;
933 }
934 
935 /**
936  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
937  * @hba: per adapter instance
938  */
939 static inline void
ufshcd_reset_intr_aggr(struct ufs_hba * hba)940 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
941 {
942 	ufshcd_writel(hba, INT_AGGR_ENABLE |
943 		      INT_AGGR_COUNTER_AND_TIMER_RESET,
944 		      REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
945 }
946 
947 /**
948  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
949  * @hba: per adapter instance
950  * @cnt: Interrupt aggregation counter threshold
951  * @tmout: Interrupt aggregation timeout value
952  */
953 static inline void
ufshcd_config_intr_aggr(struct ufs_hba * hba,u8 cnt,u8 tmout)954 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
955 {
956 	ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
957 		      INT_AGGR_COUNTER_THLD_VAL(cnt) |
958 		      INT_AGGR_TIMEOUT_VAL(tmout),
959 		      REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
960 }
961 
962 /**
963  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
964  * @hba: per adapter instance
965  */
ufshcd_disable_intr_aggr(struct ufs_hba * hba)966 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
967 {
968 	ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
969 }
970 
971 /**
972  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
973  *			When run-stop registers are set to 1, it indicates the
974  *			host controller that it can process the requests
975  * @hba: per adapter instance
976  */
ufshcd_enable_run_stop_reg(struct ufs_hba * hba)977 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
978 {
979 	ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
980 		      REG_UTP_TASK_REQ_LIST_RUN_STOP);
981 	ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
982 		      REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
983 }
984 
985 /**
986  * ufshcd_hba_start - Start controller initialization sequence
987  * @hba: per adapter instance
988  */
ufshcd_hba_start(struct ufs_hba * hba)989 static inline void ufshcd_hba_start(struct ufs_hba *hba)
990 {
991 	u32 val = CONTROLLER_ENABLE;
992 
993 	if (ufshcd_crypto_enable(hba))
994 		val |= CRYPTO_GENERAL_ENABLE;
995 
996 	ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
997 }
998 
999 /**
1000  * ufshcd_is_hba_active - Get controller state
1001  * @hba: per adapter instance
1002  *
1003  * Return: true if and only if the controller is active.
1004  */
ufshcd_is_hba_active(struct ufs_hba * hba)1005 bool ufshcd_is_hba_active(struct ufs_hba *hba)
1006 {
1007 	return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE;
1008 }
1009 EXPORT_SYMBOL_GPL(ufshcd_is_hba_active);
1010 
1011 /**
1012  * ufshcd_pm_qos_init - initialize PM QoS request
1013  * @hba: per adapter instance
1014  */
ufshcd_pm_qos_init(struct ufs_hba * hba)1015 void ufshcd_pm_qos_init(struct ufs_hba *hba)
1016 {
1017 
1018 	if (hba->pm_qos_enabled)
1019 		return;
1020 
1021 	cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE);
1022 
1023 	if (cpu_latency_qos_request_active(&hba->pm_qos_req))
1024 		hba->pm_qos_enabled = true;
1025 }
1026 
1027 /**
1028  * ufshcd_pm_qos_exit - remove request from PM QoS
1029  * @hba: per adapter instance
1030  */
ufshcd_pm_qos_exit(struct ufs_hba * hba)1031 void ufshcd_pm_qos_exit(struct ufs_hba *hba)
1032 {
1033 	if (!hba->pm_qos_enabled)
1034 		return;
1035 
1036 	cpu_latency_qos_remove_request(&hba->pm_qos_req);
1037 	hba->pm_qos_enabled = false;
1038 }
1039 
1040 /**
1041  * ufshcd_pm_qos_update - update PM QoS request
1042  * @hba: per adapter instance
1043  * @on: If True, vote for perf PM QoS mode otherwise power save mode
1044  */
ufshcd_pm_qos_update(struct ufs_hba * hba,bool on)1045 static void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on)
1046 {
1047 	if (!hba->pm_qos_enabled)
1048 		return;
1049 
1050 	cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE);
1051 }
1052 
1053 /**
1054  * ufshcd_set_clk_freq - set UFS controller clock frequencies
1055  * @hba: per adapter instance
1056  * @scale_up: If True, set max possible frequency othewise set low frequency
1057  *
1058  * Return: 0 if successful; < 0 upon failure.
1059  */
ufshcd_set_clk_freq(struct ufs_hba * hba,bool scale_up)1060 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
1061 {
1062 	int ret = 0;
1063 	struct ufs_clk_info *clki;
1064 	struct list_head *head = &hba->clk_list_head;
1065 
1066 	if (list_empty(head))
1067 		goto out;
1068 
1069 	list_for_each_entry(clki, head, list) {
1070 		if (!IS_ERR_OR_NULL(clki->clk)) {
1071 			if (scale_up && clki->max_freq) {
1072 				if (clki->curr_freq == clki->max_freq)
1073 					continue;
1074 
1075 				ret = clk_set_rate(clki->clk, clki->max_freq);
1076 				if (ret) {
1077 					dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1078 						__func__, clki->name,
1079 						clki->max_freq, ret);
1080 					break;
1081 				}
1082 				trace_ufshcd_clk_scaling(dev_name(hba->dev),
1083 						"scaled up", clki->name,
1084 						clki->curr_freq,
1085 						clki->max_freq);
1086 
1087 				clki->curr_freq = clki->max_freq;
1088 
1089 			} else if (!scale_up && clki->min_freq) {
1090 				if (clki->curr_freq == clki->min_freq)
1091 					continue;
1092 
1093 				ret = clk_set_rate(clki->clk, clki->min_freq);
1094 				if (ret) {
1095 					dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1096 						__func__, clki->name,
1097 						clki->min_freq, ret);
1098 					break;
1099 				}
1100 				trace_ufshcd_clk_scaling(dev_name(hba->dev),
1101 						"scaled down", clki->name,
1102 						clki->curr_freq,
1103 						clki->min_freq);
1104 				clki->curr_freq = clki->min_freq;
1105 			}
1106 		}
1107 		dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1108 				clki->name, clk_get_rate(clki->clk));
1109 	}
1110 
1111 out:
1112 	return ret;
1113 }
1114 
ufshcd_opp_config_clks(struct device * dev,struct opp_table * opp_table,struct dev_pm_opp * opp,void * data,bool scaling_down)1115 int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table,
1116 			   struct dev_pm_opp *opp, void *data,
1117 			   bool scaling_down)
1118 {
1119 	struct ufs_hba *hba = dev_get_drvdata(dev);
1120 	struct list_head *head = &hba->clk_list_head;
1121 	struct ufs_clk_info *clki;
1122 	unsigned long freq;
1123 	u8 idx = 0;
1124 	int ret;
1125 
1126 	list_for_each_entry(clki, head, list) {
1127 		if (!IS_ERR_OR_NULL(clki->clk)) {
1128 			freq = dev_pm_opp_get_freq_indexed(opp, idx++);
1129 
1130 			/* Do not set rate for clocks having frequency as 0 */
1131 			if (!freq)
1132 				continue;
1133 
1134 			ret = clk_set_rate(clki->clk, freq);
1135 			if (ret) {
1136 				dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n",
1137 					__func__, clki->name, freq, ret);
1138 				return ret;
1139 			}
1140 
1141 			trace_ufshcd_clk_scaling(dev_name(dev),
1142 				(scaling_down ? "scaled down" : "scaled up"),
1143 				clki->name, hba->clk_scaling.target_freq, freq);
1144 		}
1145 	}
1146 
1147 	return 0;
1148 }
1149 EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks);
1150 
ufshcd_opp_set_rate(struct ufs_hba * hba,unsigned long freq)1151 static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq)
1152 {
1153 	struct dev_pm_opp *opp;
1154 	int ret;
1155 
1156 	opp = dev_pm_opp_find_freq_floor_indexed(hba->dev,
1157 						 &freq, 0);
1158 	if (IS_ERR(opp))
1159 		return PTR_ERR(opp);
1160 
1161 	ret = dev_pm_opp_set_opp(hba->dev, opp);
1162 	dev_pm_opp_put(opp);
1163 
1164 	return ret;
1165 }
1166 
1167 /**
1168  * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1169  * @hba: per adapter instance
1170  * @freq: frequency to scale
1171  * @scale_up: True if scaling up and false if scaling down
1172  *
1173  * Return: 0 if successful; < 0 upon failure.
1174  */
ufshcd_scale_clks(struct ufs_hba * hba,unsigned long freq,bool scale_up)1175 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq,
1176 			     bool scale_up)
1177 {
1178 	int ret = 0;
1179 	ktime_t start = ktime_get();
1180 
1181 	ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1182 	if (ret)
1183 		goto out;
1184 
1185 	if (hba->use_pm_opp)
1186 		ret = ufshcd_opp_set_rate(hba, freq);
1187 	else
1188 		ret = ufshcd_set_clk_freq(hba, scale_up);
1189 	if (ret)
1190 		goto out;
1191 
1192 	ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1193 	if (ret) {
1194 		if (hba->use_pm_opp)
1195 			ufshcd_opp_set_rate(hba,
1196 					    hba->devfreq->previous_freq);
1197 		else
1198 			ufshcd_set_clk_freq(hba, !scale_up);
1199 		goto out;
1200 	}
1201 
1202 	ufshcd_pm_qos_update(hba, scale_up);
1203 
1204 out:
1205 	trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1206 			(scale_up ? "up" : "down"),
1207 			ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1208 	return ret;
1209 }
1210 
1211 /**
1212  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1213  * @hba: per adapter instance
1214  * @freq: frequency to scale
1215  * @scale_up: True if scaling up and false if scaling down
1216  *
1217  * Return: true if scaling is required, false otherwise.
1218  */
ufshcd_is_devfreq_scaling_required(struct ufs_hba * hba,unsigned long freq,bool scale_up)1219 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1220 					       unsigned long freq, bool scale_up)
1221 {
1222 	struct ufs_clk_info *clki;
1223 	struct list_head *head = &hba->clk_list_head;
1224 
1225 	if (list_empty(head))
1226 		return false;
1227 
1228 	if (hba->use_pm_opp)
1229 		return freq != hba->clk_scaling.target_freq;
1230 
1231 	list_for_each_entry(clki, head, list) {
1232 		if (!IS_ERR_OR_NULL(clki->clk)) {
1233 			if (scale_up && clki->max_freq) {
1234 				if (clki->curr_freq == clki->max_freq)
1235 					continue;
1236 				return true;
1237 			} else if (!scale_up && clki->min_freq) {
1238 				if (clki->curr_freq == clki->min_freq)
1239 					continue;
1240 				return true;
1241 			}
1242 		}
1243 	}
1244 
1245 	return false;
1246 }
1247 
1248 /*
1249  * Determine the number of pending commands by counting the bits in the SCSI
1250  * device budget maps. This approach has been selected because a bit is set in
1251  * the budget map before scsi_host_queue_ready() checks the host_self_blocked
1252  * flag. The host_self_blocked flag can be modified by calling
1253  * scsi_block_requests() or scsi_unblock_requests().
1254  */
ufshcd_pending_cmds(struct ufs_hba * hba)1255 static u32 ufshcd_pending_cmds(struct ufs_hba *hba)
1256 {
1257 	const struct scsi_device *sdev;
1258 	u32 pending = 0;
1259 
1260 	lockdep_assert_held(hba->host->host_lock);
1261 	__shost_for_each_device(sdev, hba->host)
1262 		pending += sbitmap_weight(&sdev->budget_map);
1263 
1264 	return pending;
1265 }
1266 
1267 /*
1268  * Wait until all pending SCSI commands and TMFs have finished or the timeout
1269  * has expired.
1270  *
1271  * Return: 0 upon success; -EBUSY upon timeout.
1272  */
ufshcd_wait_for_doorbell_clr(struct ufs_hba * hba,u64 wait_timeout_us)1273 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1274 					u64 wait_timeout_us)
1275 {
1276 	unsigned long flags;
1277 	int ret = 0;
1278 	u32 tm_doorbell;
1279 	u32 tr_pending;
1280 	bool timeout = false, do_last_check = false;
1281 	ktime_t start;
1282 
1283 	ufshcd_hold(hba);
1284 	spin_lock_irqsave(hba->host->host_lock, flags);
1285 	/*
1286 	 * Wait for all the outstanding tasks/transfer requests.
1287 	 * Verify by checking the doorbell registers are clear.
1288 	 */
1289 	start = ktime_get();
1290 	do {
1291 		if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1292 			ret = -EBUSY;
1293 			goto out;
1294 		}
1295 
1296 		tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1297 		tr_pending = ufshcd_pending_cmds(hba);
1298 		if (!tm_doorbell && !tr_pending) {
1299 			timeout = false;
1300 			break;
1301 		} else if (do_last_check) {
1302 			break;
1303 		}
1304 
1305 		spin_unlock_irqrestore(hba->host->host_lock, flags);
1306 		io_schedule_timeout(msecs_to_jiffies(20));
1307 		if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1308 		    wait_timeout_us) {
1309 			timeout = true;
1310 			/*
1311 			 * We might have scheduled out for long time so make
1312 			 * sure to check if doorbells are cleared by this time
1313 			 * or not.
1314 			 */
1315 			do_last_check = true;
1316 		}
1317 		spin_lock_irqsave(hba->host->host_lock, flags);
1318 	} while (tm_doorbell || tr_pending);
1319 
1320 	if (timeout) {
1321 		dev_err(hba->dev,
1322 			"%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1323 			__func__, tm_doorbell, tr_pending);
1324 		ret = -EBUSY;
1325 	}
1326 out:
1327 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1328 	ufshcd_release(hba);
1329 	return ret;
1330 }
1331 
1332 /**
1333  * ufshcd_scale_gear - scale up/down UFS gear
1334  * @hba: per adapter instance
1335  * @scale_up: True for scaling up gear and false for scaling down
1336  *
1337  * Return: 0 for success; -EBUSY if scaling can't happen at this time;
1338  * non-zero for any other errors.
1339  */
ufshcd_scale_gear(struct ufs_hba * hba,bool scale_up)1340 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1341 {
1342 	int ret = 0;
1343 	struct ufs_pa_layer_attr new_pwr_info;
1344 
1345 	if (scale_up) {
1346 		memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info,
1347 		       sizeof(struct ufs_pa_layer_attr));
1348 	} else {
1349 		memcpy(&new_pwr_info, &hba->pwr_info,
1350 		       sizeof(struct ufs_pa_layer_attr));
1351 
1352 		if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1353 		    hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1354 			/* save the current power mode */
1355 			memcpy(&hba->clk_scaling.saved_pwr_info,
1356 				&hba->pwr_info,
1357 				sizeof(struct ufs_pa_layer_attr));
1358 
1359 			/* scale down gear */
1360 			new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1361 			new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1362 		}
1363 	}
1364 
1365 	/* check if the power mode needs to be changed or not? */
1366 	ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1367 	if (ret)
1368 		dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1369 			__func__, ret,
1370 			hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1371 			new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1372 
1373 	return ret;
1374 }
1375 
1376 /*
1377  * Wait until all pending SCSI commands and TMFs have finished or the timeout
1378  * has expired.
1379  *
1380  * Return: 0 upon success; -EBUSY upon timeout.
1381  */
ufshcd_clock_scaling_prepare(struct ufs_hba * hba,u64 timeout_us)1382 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us)
1383 {
1384 	int ret = 0;
1385 	/*
1386 	 * make sure that there are no outstanding requests when
1387 	 * clock scaling is in progress
1388 	 */
1389 	blk_mq_quiesce_tagset(&hba->host->tag_set);
1390 	mutex_lock(&hba->wb_mutex);
1391 	down_write(&hba->clk_scaling_lock);
1392 
1393 	if (!hba->clk_scaling.is_allowed ||
1394 	    ufshcd_wait_for_doorbell_clr(hba, timeout_us)) {
1395 		ret = -EBUSY;
1396 		up_write(&hba->clk_scaling_lock);
1397 		mutex_unlock(&hba->wb_mutex);
1398 		blk_mq_unquiesce_tagset(&hba->host->tag_set);
1399 		goto out;
1400 	}
1401 
1402 	/* let's not get into low power until clock scaling is completed */
1403 	ufshcd_hold(hba);
1404 
1405 out:
1406 	return ret;
1407 }
1408 
ufshcd_clock_scaling_unprepare(struct ufs_hba * hba,int err,bool scale_up)1409 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool scale_up)
1410 {
1411 	up_write(&hba->clk_scaling_lock);
1412 
1413 	/* Enable Write Booster if we have scaled up else disable it */
1414 	if (ufshcd_enable_wb_if_scaling_up(hba) && !err)
1415 		ufshcd_wb_toggle(hba, scale_up);
1416 
1417 	mutex_unlock(&hba->wb_mutex);
1418 
1419 	blk_mq_unquiesce_tagset(&hba->host->tag_set);
1420 	ufshcd_release(hba);
1421 }
1422 
1423 /**
1424  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1425  * @hba: per adapter instance
1426  * @freq: frequency to scale
1427  * @scale_up: True for scaling up and false for scalin down
1428  *
1429  * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero
1430  * for any other errors.
1431  */
ufshcd_devfreq_scale(struct ufs_hba * hba,unsigned long freq,bool scale_up)1432 static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq,
1433 				bool scale_up)
1434 {
1435 	int ret = 0;
1436 
1437 	ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC);
1438 	if (ret)
1439 		return ret;
1440 
1441 	/* scale down the gear before scaling down clocks */
1442 	if (!scale_up) {
1443 		ret = ufshcd_scale_gear(hba, false);
1444 		if (ret)
1445 			goto out_unprepare;
1446 	}
1447 
1448 	ret = ufshcd_scale_clks(hba, freq, scale_up);
1449 	if (ret) {
1450 		if (!scale_up)
1451 			ufshcd_scale_gear(hba, true);
1452 		goto out_unprepare;
1453 	}
1454 
1455 	/* scale up the gear after scaling up clocks */
1456 	if (scale_up) {
1457 		ret = ufshcd_scale_gear(hba, true);
1458 		if (ret) {
1459 			ufshcd_scale_clks(hba, hba->devfreq->previous_freq,
1460 					  false);
1461 			goto out_unprepare;
1462 		}
1463 	}
1464 
1465 out_unprepare:
1466 	ufshcd_clock_scaling_unprepare(hba, ret, scale_up);
1467 	return ret;
1468 }
1469 
ufshcd_clk_scaling_suspend_work(struct work_struct * work)1470 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1471 {
1472 	struct ufs_hba *hba = container_of(work, struct ufs_hba,
1473 					   clk_scaling.suspend_work);
1474 	unsigned long irq_flags;
1475 
1476 	spin_lock_irqsave(hba->host->host_lock, irq_flags);
1477 	if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1478 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1479 		return;
1480 	}
1481 	hba->clk_scaling.is_suspended = true;
1482 	hba->clk_scaling.window_start_t = 0;
1483 	spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1484 
1485 	devfreq_suspend_device(hba->devfreq);
1486 }
1487 
ufshcd_clk_scaling_resume_work(struct work_struct * work)1488 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1489 {
1490 	struct ufs_hba *hba = container_of(work, struct ufs_hba,
1491 					   clk_scaling.resume_work);
1492 	unsigned long irq_flags;
1493 
1494 	spin_lock_irqsave(hba->host->host_lock, irq_flags);
1495 	if (!hba->clk_scaling.is_suspended) {
1496 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1497 		return;
1498 	}
1499 	hba->clk_scaling.is_suspended = false;
1500 	spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1501 
1502 	devfreq_resume_device(hba->devfreq);
1503 }
1504 
ufshcd_devfreq_target(struct device * dev,unsigned long * freq,u32 flags)1505 static int ufshcd_devfreq_target(struct device *dev,
1506 				unsigned long *freq, u32 flags)
1507 {
1508 	int ret = 0;
1509 	struct ufs_hba *hba = dev_get_drvdata(dev);
1510 	ktime_t start;
1511 	bool scale_up = false, sched_clk_scaling_suspend_work = false;
1512 	struct list_head *clk_list = &hba->clk_list_head;
1513 	struct ufs_clk_info *clki;
1514 	unsigned long irq_flags;
1515 
1516 	if (!ufshcd_is_clkscaling_supported(hba))
1517 		return -EINVAL;
1518 
1519 	if (hba->use_pm_opp) {
1520 		struct dev_pm_opp *opp;
1521 
1522 		/* Get the recommended frequency from OPP framework */
1523 		opp = devfreq_recommended_opp(dev, freq, flags);
1524 		if (IS_ERR(opp))
1525 			return PTR_ERR(opp);
1526 
1527 		dev_pm_opp_put(opp);
1528 	} else {
1529 		/* Override with the closest supported frequency */
1530 		clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info,
1531 					list);
1532 		*freq =	(unsigned long) clk_round_rate(clki->clk, *freq);
1533 	}
1534 
1535 	spin_lock_irqsave(hba->host->host_lock, irq_flags);
1536 	if (ufshcd_eh_in_progress(hba)) {
1537 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1538 		return 0;
1539 	}
1540 
1541 	/* Skip scaling clock when clock scaling is suspended */
1542 	if (hba->clk_scaling.is_suspended) {
1543 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1544 		dev_warn(hba->dev, "clock scaling is suspended, skip");
1545 		return 0;
1546 	}
1547 
1548 	if (!hba->clk_scaling.active_reqs)
1549 		sched_clk_scaling_suspend_work = true;
1550 
1551 	if (list_empty(clk_list)) {
1552 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1553 		goto out;
1554 	}
1555 
1556 	/* Decide based on the target or rounded-off frequency and update */
1557 	if (hba->use_pm_opp)
1558 		scale_up = *freq > hba->clk_scaling.target_freq;
1559 	else
1560 		scale_up = *freq == clki->max_freq;
1561 
1562 	if (!hba->use_pm_opp && !scale_up)
1563 		*freq = clki->min_freq;
1564 
1565 	/* Update the frequency */
1566 	if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) {
1567 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1568 		ret = 0;
1569 		goto out; /* no state change required */
1570 	}
1571 	spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1572 
1573 	start = ktime_get();
1574 	ret = ufshcd_devfreq_scale(hba, *freq, scale_up);
1575 	if (!ret)
1576 		hba->clk_scaling.target_freq = *freq;
1577 
1578 	trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1579 		(scale_up ? "up" : "down"),
1580 		ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1581 
1582 out:
1583 	if (sched_clk_scaling_suspend_work &&
1584 			(!scale_up || hba->clk_scaling.suspend_on_no_request))
1585 		queue_work(hba->clk_scaling.workq,
1586 			   &hba->clk_scaling.suspend_work);
1587 
1588 	return ret;
1589 }
1590 
ufshcd_devfreq_get_dev_status(struct device * dev,struct devfreq_dev_status * stat)1591 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1592 		struct devfreq_dev_status *stat)
1593 {
1594 	struct ufs_hba *hba = dev_get_drvdata(dev);
1595 	struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1596 	unsigned long flags;
1597 	ktime_t curr_t;
1598 
1599 	if (!ufshcd_is_clkscaling_supported(hba))
1600 		return -EINVAL;
1601 
1602 	memset(stat, 0, sizeof(*stat));
1603 
1604 	spin_lock_irqsave(hba->host->host_lock, flags);
1605 	curr_t = ktime_get();
1606 	if (!scaling->window_start_t)
1607 		goto start_window;
1608 
1609 	/*
1610 	 * If current frequency is 0, then the ondemand governor considers
1611 	 * there's no initial frequency set. And it always requests to set
1612 	 * to max. frequency.
1613 	 */
1614 	if (hba->use_pm_opp) {
1615 		stat->current_frequency = hba->clk_scaling.target_freq;
1616 	} else {
1617 		struct list_head *clk_list = &hba->clk_list_head;
1618 		struct ufs_clk_info *clki;
1619 
1620 		clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1621 		stat->current_frequency = clki->curr_freq;
1622 	}
1623 
1624 	if (scaling->is_busy_started)
1625 		scaling->tot_busy_t += ktime_us_delta(curr_t,
1626 				scaling->busy_start_t);
1627 	stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1628 	stat->busy_time = scaling->tot_busy_t;
1629 start_window:
1630 	scaling->window_start_t = curr_t;
1631 	scaling->tot_busy_t = 0;
1632 
1633 	if (scaling->active_reqs) {
1634 		scaling->busy_start_t = curr_t;
1635 		scaling->is_busy_started = true;
1636 	} else {
1637 		scaling->busy_start_t = 0;
1638 		scaling->is_busy_started = false;
1639 	}
1640 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1641 	return 0;
1642 }
1643 
ufshcd_devfreq_init(struct ufs_hba * hba)1644 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1645 {
1646 	struct list_head *clk_list = &hba->clk_list_head;
1647 	struct ufs_clk_info *clki;
1648 	struct devfreq *devfreq;
1649 	int ret;
1650 
1651 	/* Skip devfreq if we don't have any clocks in the list */
1652 	if (list_empty(clk_list))
1653 		return 0;
1654 
1655 	if (!hba->use_pm_opp) {
1656 		clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1657 		dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1658 		dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1659 	}
1660 
1661 	ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1662 					 &hba->vps->ondemand_data);
1663 	devfreq = devfreq_add_device(hba->dev,
1664 			&hba->vps->devfreq_profile,
1665 			DEVFREQ_GOV_SIMPLE_ONDEMAND,
1666 			&hba->vps->ondemand_data);
1667 	if (IS_ERR(devfreq)) {
1668 		ret = PTR_ERR(devfreq);
1669 		dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1670 
1671 		if (!hba->use_pm_opp) {
1672 			dev_pm_opp_remove(hba->dev, clki->min_freq);
1673 			dev_pm_opp_remove(hba->dev, clki->max_freq);
1674 		}
1675 		return ret;
1676 	}
1677 
1678 	hba->devfreq = devfreq;
1679 
1680 	return 0;
1681 }
1682 
ufshcd_devfreq_remove(struct ufs_hba * hba)1683 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1684 {
1685 	struct list_head *clk_list = &hba->clk_list_head;
1686 
1687 	if (!hba->devfreq)
1688 		return;
1689 
1690 	devfreq_remove_device(hba->devfreq);
1691 	hba->devfreq = NULL;
1692 
1693 	if (!hba->use_pm_opp) {
1694 		struct ufs_clk_info *clki;
1695 
1696 		clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1697 		dev_pm_opp_remove(hba->dev, clki->min_freq);
1698 		dev_pm_opp_remove(hba->dev, clki->max_freq);
1699 	}
1700 }
1701 
ufshcd_suspend_clkscaling(struct ufs_hba * hba)1702 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1703 {
1704 	unsigned long flags;
1705 	bool suspend = false;
1706 
1707 	cancel_work_sync(&hba->clk_scaling.suspend_work);
1708 	cancel_work_sync(&hba->clk_scaling.resume_work);
1709 
1710 	spin_lock_irqsave(hba->host->host_lock, flags);
1711 	if (!hba->clk_scaling.is_suspended) {
1712 		suspend = true;
1713 		hba->clk_scaling.is_suspended = true;
1714 		hba->clk_scaling.window_start_t = 0;
1715 	}
1716 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1717 
1718 	if (suspend)
1719 		devfreq_suspend_device(hba->devfreq);
1720 }
1721 
ufshcd_resume_clkscaling(struct ufs_hba * hba)1722 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1723 {
1724 	unsigned long flags;
1725 	bool resume = false;
1726 
1727 	spin_lock_irqsave(hba->host->host_lock, flags);
1728 	if (hba->clk_scaling.is_suspended) {
1729 		resume = true;
1730 		hba->clk_scaling.is_suspended = false;
1731 	}
1732 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1733 
1734 	if (resume)
1735 		devfreq_resume_device(hba->devfreq);
1736 }
1737 
ufshcd_clkscale_enable_show(struct device * dev,struct device_attribute * attr,char * buf)1738 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1739 		struct device_attribute *attr, char *buf)
1740 {
1741 	struct ufs_hba *hba = dev_get_drvdata(dev);
1742 
1743 	return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
1744 }
1745 
ufshcd_clkscale_enable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1746 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1747 		struct device_attribute *attr, const char *buf, size_t count)
1748 {
1749 	struct ufs_hba *hba = dev_get_drvdata(dev);
1750 	u32 value;
1751 	int err = 0;
1752 
1753 	if (kstrtou32(buf, 0, &value))
1754 		return -EINVAL;
1755 
1756 	down(&hba->host_sem);
1757 	if (!ufshcd_is_user_access_allowed(hba)) {
1758 		err = -EBUSY;
1759 		goto out;
1760 	}
1761 
1762 	value = !!value;
1763 	if (value == hba->clk_scaling.is_enabled)
1764 		goto out;
1765 
1766 	ufshcd_rpm_get_sync(hba);
1767 	ufshcd_hold(hba);
1768 
1769 	hba->clk_scaling.is_enabled = value;
1770 
1771 	if (value) {
1772 		ufshcd_resume_clkscaling(hba);
1773 	} else {
1774 		ufshcd_suspend_clkscaling(hba);
1775 		err = ufshcd_devfreq_scale(hba, ULONG_MAX, true);
1776 		if (err)
1777 			dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1778 					__func__, err);
1779 	}
1780 
1781 	ufshcd_release(hba);
1782 	ufshcd_rpm_put_sync(hba);
1783 out:
1784 	up(&hba->host_sem);
1785 	return err ? err : count;
1786 }
1787 
ufshcd_init_clk_scaling_sysfs(struct ufs_hba * hba)1788 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
1789 {
1790 	hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1791 	hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1792 	sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1793 	hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1794 	hba->clk_scaling.enable_attr.attr.mode = 0644;
1795 	if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1796 		dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1797 }
1798 
ufshcd_remove_clk_scaling_sysfs(struct ufs_hba * hba)1799 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1800 {
1801 	if (hba->clk_scaling.enable_attr.attr.name)
1802 		device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1803 }
1804 
ufshcd_init_clk_scaling(struct ufs_hba * hba)1805 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1806 {
1807 	if (!ufshcd_is_clkscaling_supported(hba))
1808 		return;
1809 
1810 	if (!hba->clk_scaling.min_gear)
1811 		hba->clk_scaling.min_gear = UFS_HS_G1;
1812 
1813 	INIT_WORK(&hba->clk_scaling.suspend_work,
1814 		  ufshcd_clk_scaling_suspend_work);
1815 	INIT_WORK(&hba->clk_scaling.resume_work,
1816 		  ufshcd_clk_scaling_resume_work);
1817 
1818 	hba->clk_scaling.workq = alloc_ordered_workqueue(
1819 		"ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no);
1820 
1821 	hba->clk_scaling.is_initialized = true;
1822 }
1823 
ufshcd_exit_clk_scaling(struct ufs_hba * hba)1824 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1825 {
1826 	if (!hba->clk_scaling.is_initialized)
1827 		return;
1828 
1829 	ufshcd_remove_clk_scaling_sysfs(hba);
1830 	destroy_workqueue(hba->clk_scaling.workq);
1831 	ufshcd_devfreq_remove(hba);
1832 	hba->clk_scaling.is_initialized = false;
1833 }
1834 
ufshcd_ungate_work(struct work_struct * work)1835 static void ufshcd_ungate_work(struct work_struct *work)
1836 {
1837 	int ret;
1838 	unsigned long flags;
1839 	struct ufs_hba *hba = container_of(work, struct ufs_hba,
1840 			clk_gating.ungate_work);
1841 
1842 	cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1843 
1844 	spin_lock_irqsave(hba->host->host_lock, flags);
1845 	if (hba->clk_gating.state == CLKS_ON) {
1846 		spin_unlock_irqrestore(hba->host->host_lock, flags);
1847 		return;
1848 	}
1849 
1850 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1851 	ufshcd_hba_vreg_set_hpm(hba);
1852 	ufshcd_setup_clocks(hba, true);
1853 
1854 	ufshcd_enable_irq(hba);
1855 
1856 	/* Exit from hibern8 */
1857 	if (ufshcd_can_hibern8_during_gating(hba)) {
1858 		/* Prevent gating in this path */
1859 		hba->clk_gating.is_suspended = true;
1860 		if (ufshcd_is_link_hibern8(hba)) {
1861 			ret = ufshcd_uic_hibern8_exit(hba);
1862 			if (ret)
1863 				dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1864 					__func__, ret);
1865 			else
1866 				ufshcd_set_link_active(hba);
1867 		}
1868 		hba->clk_gating.is_suspended = false;
1869 	}
1870 }
1871 
1872 /**
1873  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1874  * Also, exit from hibern8 mode and set the link as active.
1875  * @hba: per adapter instance
1876  */
ufshcd_hold(struct ufs_hba * hba)1877 void ufshcd_hold(struct ufs_hba *hba)
1878 {
1879 	bool flush_result;
1880 	unsigned long flags;
1881 
1882 	if (!ufshcd_is_clkgating_allowed(hba) ||
1883 	    !hba->clk_gating.is_initialized)
1884 		return;
1885 	spin_lock_irqsave(hba->host->host_lock, flags);
1886 	hba->clk_gating.active_reqs++;
1887 
1888 start:
1889 	switch (hba->clk_gating.state) {
1890 	case CLKS_ON:
1891 		/*
1892 		 * Wait for the ungate work to complete if in progress.
1893 		 * Though the clocks may be in ON state, the link could
1894 		 * still be in hibner8 state if hibern8 is allowed
1895 		 * during clock gating.
1896 		 * Make sure we exit hibern8 state also in addition to
1897 		 * clocks being ON.
1898 		 */
1899 		if (ufshcd_can_hibern8_during_gating(hba) &&
1900 		    ufshcd_is_link_hibern8(hba)) {
1901 			spin_unlock_irqrestore(hba->host->host_lock, flags);
1902 			flush_result = flush_work(&hba->clk_gating.ungate_work);
1903 			if (hba->clk_gating.is_suspended && !flush_result)
1904 				return;
1905 			spin_lock_irqsave(hba->host->host_lock, flags);
1906 			goto start;
1907 		}
1908 		break;
1909 	case REQ_CLKS_OFF:
1910 		if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1911 			hba->clk_gating.state = CLKS_ON;
1912 			trace_ufshcd_clk_gating(dev_name(hba->dev),
1913 						hba->clk_gating.state);
1914 			break;
1915 		}
1916 		/*
1917 		 * If we are here, it means gating work is either done or
1918 		 * currently running. Hence, fall through to cancel gating
1919 		 * work and to enable clocks.
1920 		 */
1921 		fallthrough;
1922 	case CLKS_OFF:
1923 		hba->clk_gating.state = REQ_CLKS_ON;
1924 		trace_ufshcd_clk_gating(dev_name(hba->dev),
1925 					hba->clk_gating.state);
1926 		queue_work(hba->clk_gating.clk_gating_workq,
1927 			   &hba->clk_gating.ungate_work);
1928 		/*
1929 		 * fall through to check if we should wait for this
1930 		 * work to be done or not.
1931 		 */
1932 		fallthrough;
1933 	case REQ_CLKS_ON:
1934 		spin_unlock_irqrestore(hba->host->host_lock, flags);
1935 		flush_work(&hba->clk_gating.ungate_work);
1936 		/* Make sure state is CLKS_ON before returning */
1937 		spin_lock_irqsave(hba->host->host_lock, flags);
1938 		goto start;
1939 	default:
1940 		dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1941 				__func__, hba->clk_gating.state);
1942 		break;
1943 	}
1944 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1945 }
1946 EXPORT_SYMBOL_GPL(ufshcd_hold);
1947 
ufshcd_gate_work(struct work_struct * work)1948 static void ufshcd_gate_work(struct work_struct *work)
1949 {
1950 	struct ufs_hba *hba = container_of(work, struct ufs_hba,
1951 			clk_gating.gate_work.work);
1952 	unsigned long flags;
1953 	int ret;
1954 
1955 	spin_lock_irqsave(hba->host->host_lock, flags);
1956 	/*
1957 	 * In case you are here to cancel this work the gating state
1958 	 * would be marked as REQ_CLKS_ON. In this case save time by
1959 	 * skipping the gating work and exit after changing the clock
1960 	 * state to CLKS_ON.
1961 	 */
1962 	if (hba->clk_gating.is_suspended ||
1963 		(hba->clk_gating.state != REQ_CLKS_OFF)) {
1964 		hba->clk_gating.state = CLKS_ON;
1965 		trace_ufshcd_clk_gating(dev_name(hba->dev),
1966 					hba->clk_gating.state);
1967 		goto rel_lock;
1968 	}
1969 
1970 	if (ufshcd_is_ufs_dev_busy(hba) || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
1971 		goto rel_lock;
1972 
1973 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1974 
1975 	/* put the link into hibern8 mode before turning off clocks */
1976 	if (ufshcd_can_hibern8_during_gating(hba)) {
1977 		ret = ufshcd_uic_hibern8_enter(hba);
1978 		if (ret) {
1979 			hba->clk_gating.state = CLKS_ON;
1980 			dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1981 					__func__, ret);
1982 			trace_ufshcd_clk_gating(dev_name(hba->dev),
1983 						hba->clk_gating.state);
1984 			goto out;
1985 		}
1986 		ufshcd_set_link_hibern8(hba);
1987 	}
1988 
1989 	ufshcd_disable_irq(hba);
1990 
1991 	ufshcd_setup_clocks(hba, false);
1992 
1993 	/* Put the host controller in low power mode if possible */
1994 	ufshcd_hba_vreg_set_lpm(hba);
1995 	/*
1996 	 * In case you are here to cancel this work the gating state
1997 	 * would be marked as REQ_CLKS_ON. In this case keep the state
1998 	 * as REQ_CLKS_ON which would anyway imply that clocks are off
1999 	 * and a request to turn them on is pending. By doing this way,
2000 	 * we keep the state machine in tact and this would ultimately
2001 	 * prevent from doing cancel work multiple times when there are
2002 	 * new requests arriving before the current cancel work is done.
2003 	 */
2004 	spin_lock_irqsave(hba->host->host_lock, flags);
2005 	if (hba->clk_gating.state == REQ_CLKS_OFF) {
2006 		hba->clk_gating.state = CLKS_OFF;
2007 		trace_ufshcd_clk_gating(dev_name(hba->dev),
2008 					hba->clk_gating.state);
2009 	}
2010 rel_lock:
2011 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2012 out:
2013 	return;
2014 }
2015 
2016 /* host lock must be held before calling this variant */
__ufshcd_release(struct ufs_hba * hba)2017 static void __ufshcd_release(struct ufs_hba *hba)
2018 {
2019 	if (!ufshcd_is_clkgating_allowed(hba))
2020 		return;
2021 
2022 	hba->clk_gating.active_reqs--;
2023 
2024 	if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
2025 	    hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
2026 	    hba->outstanding_tasks || !hba->clk_gating.is_initialized ||
2027 	    hba->active_uic_cmd || hba->uic_async_done ||
2028 	    hba->clk_gating.state == CLKS_OFF)
2029 		return;
2030 
2031 	hba->clk_gating.state = REQ_CLKS_OFF;
2032 	trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
2033 	queue_delayed_work(hba->clk_gating.clk_gating_workq,
2034 			   &hba->clk_gating.gate_work,
2035 			   msecs_to_jiffies(hba->clk_gating.delay_ms));
2036 }
2037 
ufshcd_release(struct ufs_hba * hba)2038 void ufshcd_release(struct ufs_hba *hba)
2039 {
2040 	unsigned long flags;
2041 
2042 	spin_lock_irqsave(hba->host->host_lock, flags);
2043 	__ufshcd_release(hba);
2044 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2045 }
2046 EXPORT_SYMBOL_GPL(ufshcd_release);
2047 
ufshcd_clkgate_delay_show(struct device * dev,struct device_attribute * attr,char * buf)2048 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
2049 		struct device_attribute *attr, char *buf)
2050 {
2051 	struct ufs_hba *hba = dev_get_drvdata(dev);
2052 
2053 	return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
2054 }
2055 
ufshcd_clkgate_delay_set(struct device * dev,unsigned long value)2056 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value)
2057 {
2058 	struct ufs_hba *hba = dev_get_drvdata(dev);
2059 	unsigned long flags;
2060 
2061 	spin_lock_irqsave(hba->host->host_lock, flags);
2062 	hba->clk_gating.delay_ms = value;
2063 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2064 }
2065 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set);
2066 
ufshcd_clkgate_delay_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2067 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
2068 		struct device_attribute *attr, const char *buf, size_t count)
2069 {
2070 	unsigned long value;
2071 
2072 	if (kstrtoul(buf, 0, &value))
2073 		return -EINVAL;
2074 
2075 	ufshcd_clkgate_delay_set(dev, value);
2076 	return count;
2077 }
2078 
ufshcd_clkgate_enable_show(struct device * dev,struct device_attribute * attr,char * buf)2079 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
2080 		struct device_attribute *attr, char *buf)
2081 {
2082 	struct ufs_hba *hba = dev_get_drvdata(dev);
2083 
2084 	return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
2085 }
2086 
ufshcd_clkgate_enable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2087 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
2088 		struct device_attribute *attr, const char *buf, size_t count)
2089 {
2090 	struct ufs_hba *hba = dev_get_drvdata(dev);
2091 	unsigned long flags;
2092 	u32 value;
2093 
2094 	if (kstrtou32(buf, 0, &value))
2095 		return -EINVAL;
2096 
2097 	value = !!value;
2098 
2099 	spin_lock_irqsave(hba->host->host_lock, flags);
2100 	if (value == hba->clk_gating.is_enabled)
2101 		goto out;
2102 
2103 	if (value)
2104 		__ufshcd_release(hba);
2105 	else
2106 		hba->clk_gating.active_reqs++;
2107 
2108 	hba->clk_gating.is_enabled = value;
2109 out:
2110 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2111 	return count;
2112 }
2113 
ufshcd_init_clk_gating_sysfs(struct ufs_hba * hba)2114 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
2115 {
2116 	hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
2117 	hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
2118 	sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
2119 	hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
2120 	hba->clk_gating.delay_attr.attr.mode = 0644;
2121 	if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
2122 		dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
2123 
2124 	hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
2125 	hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
2126 	sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
2127 	hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
2128 	hba->clk_gating.enable_attr.attr.mode = 0644;
2129 	if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
2130 		dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
2131 }
2132 
ufshcd_remove_clk_gating_sysfs(struct ufs_hba * hba)2133 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
2134 {
2135 	if (hba->clk_gating.delay_attr.attr.name)
2136 		device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
2137 	if (hba->clk_gating.enable_attr.attr.name)
2138 		device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
2139 }
2140 
ufshcd_init_clk_gating(struct ufs_hba * hba)2141 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
2142 {
2143 	if (!ufshcd_is_clkgating_allowed(hba))
2144 		return;
2145 
2146 	hba->clk_gating.state = CLKS_ON;
2147 
2148 	hba->clk_gating.delay_ms = 150;
2149 	INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
2150 	INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
2151 
2152 	hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(
2153 		"ufs_clk_gating_%d", WQ_MEM_RECLAIM | WQ_HIGHPRI,
2154 		hba->host->host_no);
2155 
2156 	ufshcd_init_clk_gating_sysfs(hba);
2157 
2158 	hba->clk_gating.is_enabled = true;
2159 	hba->clk_gating.is_initialized = true;
2160 }
2161 
ufshcd_exit_clk_gating(struct ufs_hba * hba)2162 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
2163 {
2164 	if (!hba->clk_gating.is_initialized)
2165 		return;
2166 
2167 	ufshcd_remove_clk_gating_sysfs(hba);
2168 
2169 	/* Ungate the clock if necessary. */
2170 	ufshcd_hold(hba);
2171 	hba->clk_gating.is_initialized = false;
2172 	ufshcd_release(hba);
2173 
2174 	destroy_workqueue(hba->clk_gating.clk_gating_workq);
2175 }
2176 
ufshcd_clk_scaling_start_busy(struct ufs_hba * hba)2177 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
2178 {
2179 	bool queue_resume_work = false;
2180 	ktime_t curr_t = ktime_get();
2181 	unsigned long flags;
2182 
2183 	if (!ufshcd_is_clkscaling_supported(hba))
2184 		return;
2185 
2186 	spin_lock_irqsave(hba->host->host_lock, flags);
2187 	if (!hba->clk_scaling.active_reqs++)
2188 		queue_resume_work = true;
2189 
2190 	if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
2191 		spin_unlock_irqrestore(hba->host->host_lock, flags);
2192 		return;
2193 	}
2194 
2195 	if (queue_resume_work)
2196 		queue_work(hba->clk_scaling.workq,
2197 			   &hba->clk_scaling.resume_work);
2198 
2199 	if (!hba->clk_scaling.window_start_t) {
2200 		hba->clk_scaling.window_start_t = curr_t;
2201 		hba->clk_scaling.tot_busy_t = 0;
2202 		hba->clk_scaling.is_busy_started = false;
2203 	}
2204 
2205 	if (!hba->clk_scaling.is_busy_started) {
2206 		hba->clk_scaling.busy_start_t = curr_t;
2207 		hba->clk_scaling.is_busy_started = true;
2208 	}
2209 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2210 }
2211 
ufshcd_clk_scaling_update_busy(struct ufs_hba * hba)2212 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2213 {
2214 	struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2215 	unsigned long flags;
2216 
2217 	if (!ufshcd_is_clkscaling_supported(hba))
2218 		return;
2219 
2220 	spin_lock_irqsave(hba->host->host_lock, flags);
2221 	hba->clk_scaling.active_reqs--;
2222 	if (!scaling->active_reqs && scaling->is_busy_started) {
2223 		scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2224 					scaling->busy_start_t));
2225 		scaling->busy_start_t = 0;
2226 		scaling->is_busy_started = false;
2227 	}
2228 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2229 }
2230 
ufshcd_monitor_opcode2dir(u8 opcode)2231 static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2232 {
2233 	if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2234 		return READ;
2235 	else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2236 		return WRITE;
2237 	else
2238 		return -EINVAL;
2239 }
2240 
ufshcd_should_inform_monitor(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2241 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2242 						struct ufshcd_lrb *lrbp)
2243 {
2244 	const struct ufs_hba_monitor *m = &hba->monitor;
2245 
2246 	return (m->enabled && lrbp && lrbp->cmd &&
2247 		(!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2248 		ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2249 }
2250 
ufshcd_start_monitor(struct ufs_hba * hba,const struct ufshcd_lrb * lrbp)2251 static void ufshcd_start_monitor(struct ufs_hba *hba,
2252 				 const struct ufshcd_lrb *lrbp)
2253 {
2254 	int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2255 	unsigned long flags;
2256 
2257 	spin_lock_irqsave(hba->host->host_lock, flags);
2258 	if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2259 		hba->monitor.busy_start_ts[dir] = ktime_get();
2260 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2261 }
2262 
ufshcd_update_monitor(struct ufs_hba * hba,const struct ufshcd_lrb * lrbp)2263 static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp)
2264 {
2265 	int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2266 	unsigned long flags;
2267 
2268 	spin_lock_irqsave(hba->host->host_lock, flags);
2269 	if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2270 		const struct request *req = scsi_cmd_to_rq(lrbp->cmd);
2271 		struct ufs_hba_monitor *m = &hba->monitor;
2272 		ktime_t now, inc, lat;
2273 
2274 		now = lrbp->compl_time_stamp;
2275 		inc = ktime_sub(now, m->busy_start_ts[dir]);
2276 		m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2277 		m->nr_sec_rw[dir] += blk_rq_sectors(req);
2278 
2279 		/* Update latencies */
2280 		m->nr_req[dir]++;
2281 		lat = ktime_sub(now, lrbp->issue_time_stamp);
2282 		m->lat_sum[dir] += lat;
2283 		if (m->lat_max[dir] < lat || !m->lat_max[dir])
2284 			m->lat_max[dir] = lat;
2285 		if (m->lat_min[dir] > lat || !m->lat_min[dir])
2286 			m->lat_min[dir] = lat;
2287 
2288 		m->nr_queued[dir]--;
2289 		/* Push forward the busy start of monitor */
2290 		m->busy_start_ts[dir] = now;
2291 	}
2292 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2293 }
2294 
2295 /**
2296  * ufshcd_send_command - Send SCSI or device management commands
2297  * @hba: per adapter instance
2298  * @task_tag: Task tag of the command
2299  * @hwq: pointer to hardware queue instance
2300  */
2301 static inline
ufshcd_send_command(struct ufs_hba * hba,unsigned int task_tag,struct ufs_hw_queue * hwq)2302 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag,
2303 			 struct ufs_hw_queue *hwq)
2304 {
2305 	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2306 	unsigned long flags;
2307 
2308 	lrbp->issue_time_stamp = ktime_get();
2309 	lrbp->issue_time_stamp_local_clock = local_clock();
2310 	lrbp->compl_time_stamp = ktime_set(0, 0);
2311 	lrbp->compl_time_stamp_local_clock = 0;
2312 	ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
2313 	if (lrbp->cmd)
2314 		ufshcd_clk_scaling_start_busy(hba);
2315 	if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2316 		ufshcd_start_monitor(hba, lrbp);
2317 
2318 	if (hba->mcq_enabled) {
2319 		int utrd_size = sizeof(struct utp_transfer_req_desc);
2320 		struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr;
2321 		struct utp_transfer_req_desc *dest;
2322 
2323 		spin_lock(&hwq->sq_lock);
2324 		dest = hwq->sqe_base_addr + hwq->sq_tail_slot;
2325 		memcpy(dest, src, utrd_size);
2326 		ufshcd_inc_sq_tail(hwq);
2327 		spin_unlock(&hwq->sq_lock);
2328 	} else {
2329 		spin_lock_irqsave(&hba->outstanding_lock, flags);
2330 		if (hba->vops && hba->vops->setup_xfer_req)
2331 			hba->vops->setup_xfer_req(hba, lrbp->task_tag,
2332 						  !!lrbp->cmd);
2333 		__set_bit(lrbp->task_tag, &hba->outstanding_reqs);
2334 		ufshcd_writel(hba, 1 << lrbp->task_tag,
2335 			      REG_UTP_TRANSFER_REQ_DOOR_BELL);
2336 		spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2337 	}
2338 }
2339 
2340 /**
2341  * ufshcd_copy_sense_data - Copy sense data in case of check condition
2342  * @lrbp: pointer to local reference block
2343  */
ufshcd_copy_sense_data(struct ufshcd_lrb * lrbp)2344 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2345 {
2346 	u8 *const sense_buffer = lrbp->cmd->sense_buffer;
2347 	u16 resp_len;
2348 	int len;
2349 
2350 	resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length);
2351 	if (sense_buffer && resp_len) {
2352 		int len_to_copy;
2353 
2354 		len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2355 		len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
2356 
2357 		memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2358 		       len_to_copy);
2359 	}
2360 }
2361 
2362 /**
2363  * ufshcd_copy_query_response() - Copy the Query Response and the data
2364  * descriptor
2365  * @hba: per adapter instance
2366  * @lrbp: pointer to local reference block
2367  *
2368  * Return: 0 upon success; < 0 upon failure.
2369  */
2370 static
ufshcd_copy_query_response(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2371 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2372 {
2373 	struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2374 
2375 	memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2376 
2377 	/* Get the descriptor */
2378 	if (hba->dev_cmd.query.descriptor &&
2379 	    lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2380 		u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2381 				GENERAL_UPIU_REQUEST_SIZE;
2382 		u16 resp_len;
2383 		u16 buf_len;
2384 
2385 		/* data segment length */
2386 		resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
2387 				       .data_segment_length);
2388 		buf_len = be16_to_cpu(
2389 				hba->dev_cmd.query.request.upiu_req.length);
2390 		if (likely(buf_len >= resp_len)) {
2391 			memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2392 		} else {
2393 			dev_warn(hba->dev,
2394 				 "%s: rsp size %d is bigger than buffer size %d",
2395 				 __func__, resp_len, buf_len);
2396 			return -EINVAL;
2397 		}
2398 	}
2399 
2400 	return 0;
2401 }
2402 
2403 /**
2404  * ufshcd_hba_capabilities - Read controller capabilities
2405  * @hba: per adapter instance
2406  *
2407  * Return: 0 on success, negative on error.
2408  */
ufshcd_hba_capabilities(struct ufs_hba * hba)2409 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2410 {
2411 	int err;
2412 
2413 	hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2414 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS)
2415 		hba->capabilities &= ~MASK_64_ADDRESSING_SUPPORT;
2416 
2417 	/* nutrs and nutmrs are 0 based values */
2418 	hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1;
2419 	hba->nutmrs =
2420 	((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2421 	hba->reserved_slot = hba->nutrs - 1;
2422 
2423 	hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1;
2424 
2425 	/* Read crypto capabilities */
2426 	err = ufshcd_hba_init_crypto_capabilities(hba);
2427 	if (err) {
2428 		dev_err(hba->dev, "crypto setup failed\n");
2429 		return err;
2430 	}
2431 
2432 	/*
2433 	 * The UFSHCI 3.0 specification does not define MCQ_SUPPORT and
2434 	 * LSDB_SUPPORT, but [31:29] as reserved bits with reset value 0s, which
2435 	 * means we can simply read values regardless of version.
2436 	 */
2437 	hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities);
2438 	/*
2439 	 * 0h: legacy single doorbell support is available
2440 	 * 1h: indicate that legacy single doorbell support has been removed
2441 	 */
2442 	if (!(hba->quirks & UFSHCD_QUIRK_BROKEN_LSDBS_CAP))
2443 		hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities);
2444 	else
2445 		hba->lsdb_sup = true;
2446 
2447 	if (!hba->mcq_sup)
2448 		return 0;
2449 
2450 	hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP);
2451 	hba->ext_iid_sup = FIELD_GET(MASK_EXT_IID_SUPPORT,
2452 				     hba->mcq_capabilities);
2453 
2454 	return 0;
2455 }
2456 
2457 /**
2458  * ufshcd_ready_for_uic_cmd - Check if controller is ready
2459  *                            to accept UIC commands
2460  * @hba: per adapter instance
2461  *
2462  * Return: true on success, else false.
2463  */
ufshcd_ready_for_uic_cmd(struct ufs_hba * hba)2464 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2465 {
2466 	u32 val;
2467 	int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY,
2468 				    500, uic_cmd_timeout * 1000, false, hba,
2469 				    REG_CONTROLLER_STATUS);
2470 	return ret == 0;
2471 }
2472 
2473 /**
2474  * ufshcd_get_upmcrs - Get the power mode change request status
2475  * @hba: Pointer to adapter instance
2476  *
2477  * This function gets the UPMCRS field of HCS register
2478  *
2479  * Return: value of UPMCRS field.
2480  */
ufshcd_get_upmcrs(struct ufs_hba * hba)2481 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2482 {
2483 	return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2484 }
2485 
2486 /**
2487  * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer
2488  * @hba: per adapter instance
2489  * @uic_cmd: UIC command
2490  */
2491 static inline void
ufshcd_dispatch_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2492 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2493 {
2494 	lockdep_assert_held(&hba->uic_cmd_mutex);
2495 
2496 	WARN_ON(hba->active_uic_cmd);
2497 
2498 	hba->active_uic_cmd = uic_cmd;
2499 
2500 	/* Write Args */
2501 	ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2502 	ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2503 	ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2504 
2505 	ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
2506 
2507 	/* Write UIC Cmd */
2508 	ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2509 		      REG_UIC_COMMAND);
2510 }
2511 
2512 /**
2513  * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command
2514  * @hba: per adapter instance
2515  * @uic_cmd: UIC command
2516  *
2517  * Return: 0 only if success.
2518  */
2519 static int
ufshcd_wait_for_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2520 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2521 {
2522 	int ret;
2523 	unsigned long flags;
2524 
2525 	lockdep_assert_held(&hba->uic_cmd_mutex);
2526 
2527 	if (wait_for_completion_timeout(&uic_cmd->done,
2528 					msecs_to_jiffies(uic_cmd_timeout))) {
2529 		ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2530 	} else {
2531 		ret = -ETIMEDOUT;
2532 		dev_err(hba->dev,
2533 			"uic cmd 0x%x with arg3 0x%x completion timeout\n",
2534 			uic_cmd->command, uic_cmd->argument3);
2535 
2536 		if (!uic_cmd->cmd_active) {
2537 			dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2538 				__func__);
2539 			ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2540 		}
2541 	}
2542 
2543 	spin_lock_irqsave(hba->host->host_lock, flags);
2544 	hba->active_uic_cmd = NULL;
2545 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2546 
2547 	return ret;
2548 }
2549 
2550 /**
2551  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2552  * @hba: per adapter instance
2553  * @uic_cmd: UIC command
2554  * @completion: initialize the completion only if this is set to true
2555  *
2556  * Return: 0 only if success.
2557  */
2558 static int
__ufshcd_send_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd,bool completion)2559 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2560 		      bool completion)
2561 {
2562 	lockdep_assert_held(&hba->uic_cmd_mutex);
2563 
2564 	if (!ufshcd_ready_for_uic_cmd(hba)) {
2565 		dev_err(hba->dev,
2566 			"Controller not ready to accept UIC commands\n");
2567 		return -EIO;
2568 	}
2569 
2570 	if (completion)
2571 		init_completion(&uic_cmd->done);
2572 
2573 	uic_cmd->cmd_active = 1;
2574 	ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2575 
2576 	return 0;
2577 }
2578 
2579 /**
2580  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2581  * @hba: per adapter instance
2582  * @uic_cmd: UIC command
2583  *
2584  * Return: 0 only if success.
2585  */
ufshcd_send_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2586 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2587 {
2588 	int ret;
2589 
2590 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
2591 		return 0;
2592 
2593 	ufshcd_hold(hba);
2594 	mutex_lock(&hba->uic_cmd_mutex);
2595 	ufshcd_add_delay_before_dme_cmd(hba);
2596 
2597 	ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2598 	if (!ret)
2599 		ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2600 
2601 	mutex_unlock(&hba->uic_cmd_mutex);
2602 
2603 	ufshcd_release(hba);
2604 	return ret;
2605 }
2606 
2607 /**
2608  * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format)
2609  * @hba:	per-adapter instance
2610  * @lrbp:	pointer to local reference block
2611  * @sg_entries:	The number of sg lists actually used
2612  * @sg_list:	Pointer to SG list
2613  */
ufshcd_sgl_to_prdt(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,int sg_entries,struct scatterlist * sg_list)2614 static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries,
2615 			       struct scatterlist *sg_list)
2616 {
2617 	struct ufshcd_sg_entry *prd;
2618 	struct scatterlist *sg;
2619 	int i;
2620 
2621 	if (sg_entries) {
2622 
2623 		if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2624 			lrbp->utr_descriptor_ptr->prd_table_length =
2625 				cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba));
2626 		else
2627 			lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries);
2628 
2629 		prd = lrbp->ucd_prdt_ptr;
2630 
2631 		for_each_sg(sg_list, sg, sg_entries, i) {
2632 			const unsigned int len = sg_dma_len(sg);
2633 
2634 			/*
2635 			 * From the UFSHCI spec: "Data Byte Count (DBC): A '0'
2636 			 * based value that indicates the length, in bytes, of
2637 			 * the data block. A maximum of length of 256KB may
2638 			 * exist for any entry. Bits 1:0 of this field shall be
2639 			 * 11b to indicate Dword granularity. A value of '3'
2640 			 * indicates 4 bytes, '7' indicates 8 bytes, etc."
2641 			 */
2642 			WARN_ONCE(len > SZ_256K, "len = %#x\n", len);
2643 			prd->size = cpu_to_le32(len - 1);
2644 			prd->addr = cpu_to_le64(sg->dma_address);
2645 			prd->reserved = 0;
2646 			prd = (void *)prd + ufshcd_sg_entry_size(hba);
2647 		}
2648 	} else {
2649 		lrbp->utr_descriptor_ptr->prd_table_length = 0;
2650 	}
2651 }
2652 
2653 /**
2654  * ufshcd_map_sg - Map scatter-gather list to prdt
2655  * @hba: per adapter instance
2656  * @lrbp: pointer to local reference block
2657  *
2658  * Return: 0 in case of success, non-zero value in case of failure.
2659  */
ufshcd_map_sg(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2660 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2661 {
2662 	struct scsi_cmnd *cmd = lrbp->cmd;
2663 	int sg_segments = scsi_dma_map(cmd);
2664 
2665 	if (sg_segments < 0)
2666 		return sg_segments;
2667 
2668 	ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));
2669 
2670 	return ufshcd_crypto_fill_prdt(hba, lrbp);
2671 }
2672 
2673 /**
2674  * ufshcd_enable_intr - enable interrupts
2675  * @hba: per adapter instance
2676  * @intrs: interrupt bits
2677  */
ufshcd_enable_intr(struct ufs_hba * hba,u32 intrs)2678 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2679 {
2680 	u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2681 
2682 	set |= intrs;
2683 	ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2684 }
2685 
2686 /**
2687  * ufshcd_disable_intr - disable interrupts
2688  * @hba: per adapter instance
2689  * @intrs: interrupt bits
2690  */
ufshcd_disable_intr(struct ufs_hba * hba,u32 intrs)2691 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2692 {
2693 	u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2694 
2695 	set &= ~intrs;
2696 	ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2697 }
2698 
2699 /**
2700  * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request
2701  * descriptor according to request
2702  * @hba: per adapter instance
2703  * @lrbp: pointer to local reference block
2704  * @upiu_flags: flags required in the header
2705  * @cmd_dir: requests data direction
2706  * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments)
2707  */
2708 static void
ufshcd_prepare_req_desc_hdr(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,u8 * upiu_flags,enum dma_data_direction cmd_dir,int ehs_length)2709 ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
2710 			    u8 *upiu_flags, enum dma_data_direction cmd_dir,
2711 			    int ehs_length)
2712 {
2713 	struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2714 	struct request_desc_header *h = &req_desc->header;
2715 	enum utp_data_direction data_direction;
2716 
2717 	lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2718 
2719 	*h = (typeof(*h)){ };
2720 
2721 	if (cmd_dir == DMA_FROM_DEVICE) {
2722 		data_direction = UTP_DEVICE_TO_HOST;
2723 		*upiu_flags = UPIU_CMD_FLAGS_READ;
2724 	} else if (cmd_dir == DMA_TO_DEVICE) {
2725 		data_direction = UTP_HOST_TO_DEVICE;
2726 		*upiu_flags = UPIU_CMD_FLAGS_WRITE;
2727 	} else {
2728 		data_direction = UTP_NO_DATA_TRANSFER;
2729 		*upiu_flags = UPIU_CMD_FLAGS_NONE;
2730 	}
2731 
2732 	h->command_type = lrbp->command_type;
2733 	h->data_direction = data_direction;
2734 	h->ehs_length = ehs_length;
2735 
2736 	if (lrbp->intr_cmd)
2737 		h->interrupt = 1;
2738 
2739 	/* Prepare crypto related dwords */
2740 	ufshcd_prepare_req_desc_hdr_crypto(lrbp, h);
2741 
2742 	/*
2743 	 * assigning invalid value for command status. Controller
2744 	 * updates OCS on command completion, with the command
2745 	 * status
2746 	 */
2747 	h->ocs = OCS_INVALID_COMMAND_STATUS;
2748 
2749 	req_desc->prd_table_length = 0;
2750 }
2751 
2752 /**
2753  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2754  * for scsi commands
2755  * @lrbp: local reference block pointer
2756  * @upiu_flags: flags
2757  */
2758 static
ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb * lrbp,u8 upiu_flags)2759 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2760 {
2761 	struct scsi_cmnd *cmd = lrbp->cmd;
2762 	struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2763 	unsigned short cdb_len;
2764 
2765 	ucd_req_ptr->header = (struct utp_upiu_header){
2766 		.transaction_code = UPIU_TRANSACTION_COMMAND,
2767 		.flags = upiu_flags,
2768 		.lun = lrbp->lun,
2769 		.task_tag = lrbp->task_tag,
2770 		.command_set_type = UPIU_COMMAND_SET_TYPE_SCSI,
2771 	};
2772 
2773 	WARN_ON_ONCE(ucd_req_ptr->header.task_tag != lrbp->task_tag);
2774 
2775 	ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2776 
2777 	cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2778 	memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2779 	memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2780 
2781 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2782 }
2783 
2784 /**
2785  * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request
2786  * @hba: UFS hba
2787  * @lrbp: local reference block pointer
2788  * @upiu_flags: flags
2789  */
ufshcd_prepare_utp_query_req_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,u8 upiu_flags)2790 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2791 				struct ufshcd_lrb *lrbp, u8 upiu_flags)
2792 {
2793 	struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2794 	struct ufs_query *query = &hba->dev_cmd.query;
2795 	u16 len = be16_to_cpu(query->request.upiu_req.length);
2796 
2797 	/* Query request header */
2798 	ucd_req_ptr->header = (struct utp_upiu_header){
2799 		.transaction_code = UPIU_TRANSACTION_QUERY_REQ,
2800 		.flags = upiu_flags,
2801 		.lun = lrbp->lun,
2802 		.task_tag = lrbp->task_tag,
2803 		.query_function = query->request.query_func,
2804 		/* Data segment length only need for WRITE_DESC */
2805 		.data_segment_length =
2806 			query->request.upiu_req.opcode ==
2807 					UPIU_QUERY_OPCODE_WRITE_DESC ?
2808 				cpu_to_be16(len) :
2809 				0,
2810 	};
2811 
2812 	/* Copy the Query Request buffer as is */
2813 	memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2814 			QUERY_OSF_SIZE);
2815 
2816 	/* Copy the Descriptor */
2817 	if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2818 		memcpy(ucd_req_ptr + 1, query->descriptor, len);
2819 
2820 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2821 }
2822 
ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb * lrbp)2823 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2824 {
2825 	struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2826 
2827 	memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2828 
2829 	ucd_req_ptr->header = (struct utp_upiu_header){
2830 		.transaction_code = UPIU_TRANSACTION_NOP_OUT,
2831 		.task_tag = lrbp->task_tag,
2832 	};
2833 
2834 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2835 }
2836 
2837 /**
2838  * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2839  *			     for Device Management Purposes
2840  * @hba: per adapter instance
2841  * @lrbp: pointer to local reference block
2842  *
2843  * Return: 0 upon success; < 0 upon failure.
2844  */
ufshcd_compose_devman_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2845 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2846 				      struct ufshcd_lrb *lrbp)
2847 {
2848 	u8 upiu_flags;
2849 	int ret = 0;
2850 
2851 	ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
2852 
2853 	if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2854 		ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2855 	else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2856 		ufshcd_prepare_utp_nop_upiu(lrbp);
2857 	else
2858 		ret = -EINVAL;
2859 
2860 	return ret;
2861 }
2862 
2863 /**
2864  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2865  *			   for SCSI Purposes
2866  * @hba: per adapter instance
2867  * @lrbp: pointer to local reference block
2868  */
ufshcd_comp_scsi_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2869 static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2870 {
2871 	struct request *rq = scsi_cmd_to_rq(lrbp->cmd);
2872 	unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
2873 	u8 upiu_flags;
2874 
2875 	ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0);
2876 	if (ioprio_class == IOPRIO_CLASS_RT)
2877 		upiu_flags |= UPIU_CMD_FLAGS_CP;
2878 	ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2879 }
2880 
2881 /**
2882  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2883  * @upiu_wlun_id: UPIU W-LUN id
2884  *
2885  * Return: SCSI W-LUN id.
2886  */
ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)2887 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2888 {
2889 	return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2890 }
2891 
is_device_wlun(struct scsi_device * sdev)2892 static inline bool is_device_wlun(struct scsi_device *sdev)
2893 {
2894 	return sdev->lun ==
2895 		ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN);
2896 }
2897 
2898 /*
2899  * Associate the UFS controller queue with the default and poll HCTX types.
2900  * Initialize the mq_map[] arrays.
2901  */
ufshcd_map_queues(struct Scsi_Host * shost)2902 static void ufshcd_map_queues(struct Scsi_Host *shost)
2903 {
2904 	struct ufs_hba *hba = shost_priv(shost);
2905 	int i, queue_offset = 0;
2906 
2907 	if (!is_mcq_supported(hba)) {
2908 		hba->nr_queues[HCTX_TYPE_DEFAULT] = 1;
2909 		hba->nr_queues[HCTX_TYPE_READ] = 0;
2910 		hba->nr_queues[HCTX_TYPE_POLL] = 1;
2911 		hba->nr_hw_queues = 1;
2912 	}
2913 
2914 	for (i = 0; i < shost->nr_maps; i++) {
2915 		struct blk_mq_queue_map *map = &shost->tag_set.map[i];
2916 
2917 		map->nr_queues = hba->nr_queues[i];
2918 		if (!map->nr_queues)
2919 			continue;
2920 		map->queue_offset = queue_offset;
2921 		if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba))
2922 			map->queue_offset = 0;
2923 
2924 		blk_mq_map_queues(map);
2925 		queue_offset += map->nr_queues;
2926 	}
2927 }
2928 
ufshcd_init_lrb(struct ufs_hba * hba,struct ufshcd_lrb * lrb,int i)2929 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2930 {
2931 	struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr +
2932 		i * ufshcd_get_ucd_size(hba);
2933 	struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2934 	dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2935 		i * ufshcd_get_ucd_size(hba);
2936 	u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset);
2937 	u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset);
2938 
2939 	lrb->utr_descriptor_ptr = utrdlp + i;
2940 	lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2941 		i * sizeof(struct utp_transfer_req_desc);
2942 	lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu;
2943 	lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2944 	lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu;
2945 	lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2946 	lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table;
2947 	lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2948 }
2949 
2950 /**
2951  * ufshcd_queuecommand - main entry point for SCSI requests
2952  * @host: SCSI host pointer
2953  * @cmd: command from SCSI Midlayer
2954  *
2955  * Return: 0 for success, non-zero in case of failure.
2956  */
ufshcd_queuecommand(struct Scsi_Host * host,struct scsi_cmnd * cmd)2957 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2958 {
2959 	struct ufs_hba *hba = shost_priv(host);
2960 	int tag = scsi_cmd_to_rq(cmd)->tag;
2961 	struct ufshcd_lrb *lrbp;
2962 	int err = 0;
2963 	struct ufs_hw_queue *hwq = NULL;
2964 
2965 	switch (hba->ufshcd_state) {
2966 	case UFSHCD_STATE_OPERATIONAL:
2967 		break;
2968 	case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2969 		/*
2970 		 * SCSI error handler can call ->queuecommand() while UFS error
2971 		 * handler is in progress. Error interrupts could change the
2972 		 * state from UFSHCD_STATE_RESET to
2973 		 * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests
2974 		 * being issued in that case.
2975 		 */
2976 		if (ufshcd_eh_in_progress(hba)) {
2977 			err = SCSI_MLQUEUE_HOST_BUSY;
2978 			goto out;
2979 		}
2980 		break;
2981 	case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2982 		/*
2983 		 * pm_runtime_get_sync() is used at error handling preparation
2984 		 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2985 		 * PM ops, it can never be finished if we let SCSI layer keep
2986 		 * retrying it, which gets err handler stuck forever. Neither
2987 		 * can we let the scsi cmd pass through, because UFS is in bad
2988 		 * state, the scsi cmd may eventually time out, which will get
2989 		 * err handler blocked for too long. So, just fail the scsi cmd
2990 		 * sent from PM ops, err handler can recover PM error anyways.
2991 		 */
2992 		if (hba->pm_op_in_progress) {
2993 			hba->force_reset = true;
2994 			set_host_byte(cmd, DID_BAD_TARGET);
2995 			scsi_done(cmd);
2996 			goto out;
2997 		}
2998 		fallthrough;
2999 	case UFSHCD_STATE_RESET:
3000 		err = SCSI_MLQUEUE_HOST_BUSY;
3001 		goto out;
3002 	case UFSHCD_STATE_ERROR:
3003 		set_host_byte(cmd, DID_ERROR);
3004 		scsi_done(cmd);
3005 		goto out;
3006 	}
3007 
3008 	hba->req_abort_count = 0;
3009 
3010 	ufshcd_hold(hba);
3011 
3012 	lrbp = &hba->lrb[tag];
3013 	lrbp->cmd = cmd;
3014 	lrbp->task_tag = tag;
3015 	lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
3016 	lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
3017 
3018 	ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp);
3019 
3020 	lrbp->req_abort_skip = false;
3021 
3022 	ufshcd_comp_scsi_upiu(hba, lrbp);
3023 
3024 	err = ufshcd_map_sg(hba, lrbp);
3025 	if (err) {
3026 		ufshcd_release(hba);
3027 		goto out;
3028 	}
3029 
3030 	if (hba->mcq_enabled)
3031 		hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
3032 
3033 	ufshcd_send_command(hba, tag, hwq);
3034 
3035 out:
3036 	if (ufs_trigger_eh(hba)) {
3037 		unsigned long flags;
3038 
3039 		spin_lock_irqsave(hba->host->host_lock, flags);
3040 		ufshcd_schedule_eh_work(hba);
3041 		spin_unlock_irqrestore(hba->host->host_lock, flags);
3042 	}
3043 
3044 	return err;
3045 }
3046 
ufshcd_setup_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,enum dev_cmd_type cmd_type,u8 lun,int tag)3047 static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
3048 			     enum dev_cmd_type cmd_type, u8 lun, int tag)
3049 {
3050 	lrbp->cmd = NULL;
3051 	lrbp->task_tag = tag;
3052 	lrbp->lun = lun;
3053 	lrbp->intr_cmd = true; /* No interrupt aggregation */
3054 	ufshcd_prepare_lrbp_crypto(NULL, lrbp);
3055 	hba->dev_cmd.type = cmd_type;
3056 }
3057 
ufshcd_compose_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,enum dev_cmd_type cmd_type,int tag)3058 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
3059 		struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
3060 {
3061 	ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
3062 
3063 	return ufshcd_compose_devman_upiu(hba, lrbp);
3064 }
3065 
3066 /*
3067  * Check with the block layer if the command is inflight
3068  * @cmd: command to check.
3069  *
3070  * Return: true if command is inflight; false if not.
3071  */
ufshcd_cmd_inflight(struct scsi_cmnd * cmd)3072 bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd)
3073 {
3074 	return cmd && blk_mq_rq_state(scsi_cmd_to_rq(cmd)) == MQ_RQ_IN_FLIGHT;
3075 }
3076 
3077 /*
3078  * Clear the pending command in the controller and wait until
3079  * the controller confirms that the command has been cleared.
3080  * @hba: per adapter instance
3081  * @task_tag: The tag number of the command to be cleared.
3082  */
ufshcd_clear_cmd(struct ufs_hba * hba,u32 task_tag)3083 static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag)
3084 {
3085 	u32 mask;
3086 	unsigned long flags;
3087 	int err;
3088 
3089 	if (hba->mcq_enabled) {
3090 		/*
3091 		 * MCQ mode. Clean up the MCQ resources similar to
3092 		 * what the ufshcd_utrl_clear() does for SDB mode.
3093 		 */
3094 		err = ufshcd_mcq_sq_cleanup(hba, task_tag);
3095 		if (err) {
3096 			dev_err(hba->dev, "%s: failed tag=%d. err=%d\n",
3097 				__func__, task_tag, err);
3098 			return err;
3099 		}
3100 		return 0;
3101 	}
3102 
3103 	mask = 1U << task_tag;
3104 
3105 	/* clear outstanding transaction before retry */
3106 	spin_lock_irqsave(hba->host->host_lock, flags);
3107 	ufshcd_utrl_clear(hba, mask);
3108 	spin_unlock_irqrestore(hba->host->host_lock, flags);
3109 
3110 	/*
3111 	 * wait for h/w to clear corresponding bit in door-bell.
3112 	 * max. wait is 1 sec.
3113 	 */
3114 	return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL,
3115 					mask, ~mask, 1000, 1000);
3116 }
3117 
3118 /**
3119  * ufshcd_dev_cmd_completion() - handles device management command responses
3120  * @hba: per adapter instance
3121  * @lrbp: pointer to local reference block
3122  *
3123  * Return: 0 upon success; < 0 upon failure.
3124  */
3125 static int
ufshcd_dev_cmd_completion(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)3126 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
3127 {
3128 	enum upiu_response_transaction resp;
3129 	int err = 0;
3130 
3131 	hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
3132 	resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
3133 
3134 	switch (resp) {
3135 	case UPIU_TRANSACTION_NOP_IN:
3136 		if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
3137 			err = -EINVAL;
3138 			dev_err(hba->dev, "%s: unexpected response %x\n",
3139 					__func__, resp);
3140 		}
3141 		break;
3142 	case UPIU_TRANSACTION_QUERY_RSP: {
3143 		u8 response = lrbp->ucd_rsp_ptr->header.response;
3144 
3145 		if (response == 0)
3146 			err = ufshcd_copy_query_response(hba, lrbp);
3147 		break;
3148 	}
3149 	case UPIU_TRANSACTION_REJECT_UPIU:
3150 		/* TODO: handle Reject UPIU Response */
3151 		err = -EPERM;
3152 		dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
3153 				__func__);
3154 		break;
3155 	case UPIU_TRANSACTION_RESPONSE:
3156 		if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) {
3157 			err = -EINVAL;
3158 			dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp);
3159 		}
3160 		break;
3161 	default:
3162 		err = -EINVAL;
3163 		dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
3164 				__func__, resp);
3165 		break;
3166 	}
3167 
3168 	return err;
3169 }
3170 
ufshcd_wait_for_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,int max_timeout)3171 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
3172 		struct ufshcd_lrb *lrbp, int max_timeout)
3173 {
3174 	unsigned long time_left = msecs_to_jiffies(max_timeout);
3175 	unsigned long flags;
3176 	bool pending;
3177 	int err;
3178 
3179 retry:
3180 	time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
3181 						time_left);
3182 
3183 	if (likely(time_left)) {
3184 		/*
3185 		 * The completion handler called complete() and the caller of
3186 		 * this function still owns the @lrbp tag so the code below does
3187 		 * not trigger any race conditions.
3188 		 */
3189 		hba->dev_cmd.complete = NULL;
3190 		err = ufshcd_get_tr_ocs(lrbp, NULL);
3191 		if (!err)
3192 			err = ufshcd_dev_cmd_completion(hba, lrbp);
3193 	} else {
3194 		err = -ETIMEDOUT;
3195 		dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
3196 			__func__, lrbp->task_tag);
3197 
3198 		/* MCQ mode */
3199 		if (hba->mcq_enabled) {
3200 			/* successfully cleared the command, retry if needed */
3201 			if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0)
3202 				err = -EAGAIN;
3203 			hba->dev_cmd.complete = NULL;
3204 			return err;
3205 		}
3206 
3207 		/* SDB mode */
3208 		if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) {
3209 			/* successfully cleared the command, retry if needed */
3210 			err = -EAGAIN;
3211 			/*
3212 			 * Since clearing the command succeeded we also need to
3213 			 * clear the task tag bit from the outstanding_reqs
3214 			 * variable.
3215 			 */
3216 			spin_lock_irqsave(&hba->outstanding_lock, flags);
3217 			pending = test_bit(lrbp->task_tag,
3218 					   &hba->outstanding_reqs);
3219 			if (pending) {
3220 				hba->dev_cmd.complete = NULL;
3221 				__clear_bit(lrbp->task_tag,
3222 					    &hba->outstanding_reqs);
3223 			}
3224 			spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3225 
3226 			if (!pending) {
3227 				/*
3228 				 * The completion handler ran while we tried to
3229 				 * clear the command.
3230 				 */
3231 				time_left = 1;
3232 				goto retry;
3233 			}
3234 		} else {
3235 			dev_err(hba->dev, "%s: failed to clear tag %d\n",
3236 				__func__, lrbp->task_tag);
3237 
3238 			spin_lock_irqsave(&hba->outstanding_lock, flags);
3239 			pending = test_bit(lrbp->task_tag,
3240 					   &hba->outstanding_reqs);
3241 			if (pending)
3242 				hba->dev_cmd.complete = NULL;
3243 			spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3244 
3245 			if (!pending) {
3246 				/*
3247 				 * The completion handler ran while we tried to
3248 				 * clear the command.
3249 				 */
3250 				time_left = 1;
3251 				goto retry;
3252 			}
3253 		}
3254 	}
3255 
3256 	return err;
3257 }
3258 
ufshcd_dev_man_lock(struct ufs_hba * hba)3259 static void ufshcd_dev_man_lock(struct ufs_hba *hba)
3260 {
3261 	ufshcd_hold(hba);
3262 	mutex_lock(&hba->dev_cmd.lock);
3263 	down_read(&hba->clk_scaling_lock);
3264 }
3265 
ufshcd_dev_man_unlock(struct ufs_hba * hba)3266 static void ufshcd_dev_man_unlock(struct ufs_hba *hba)
3267 {
3268 	up_read(&hba->clk_scaling_lock);
3269 	mutex_unlock(&hba->dev_cmd.lock);
3270 	ufshcd_release(hba);
3271 }
3272 
ufshcd_issue_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,const u32 tag,int timeout)3273 static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
3274 			  const u32 tag, int timeout)
3275 {
3276 	DECLARE_COMPLETION_ONSTACK(wait);
3277 	int err;
3278 
3279 	hba->dev_cmd.complete = &wait;
3280 
3281 	ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
3282 
3283 	ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
3284 	err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
3285 
3286 	ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
3287 				    (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
3288 
3289 	return err;
3290 }
3291 
3292 /**
3293  * ufshcd_exec_dev_cmd - API for sending device management requests
3294  * @hba: UFS hba
3295  * @cmd_type: specifies the type (NOP, Query...)
3296  * @timeout: timeout in milliseconds
3297  *
3298  * Return: 0 upon success; < 0 upon failure.
3299  *
3300  * NOTE: Since there is only one available tag for device management commands,
3301  * it is expected you hold the hba->dev_cmd.lock mutex.
3302  */
ufshcd_exec_dev_cmd(struct ufs_hba * hba,enum dev_cmd_type cmd_type,int timeout)3303 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
3304 		enum dev_cmd_type cmd_type, int timeout)
3305 {
3306 	const u32 tag = hba->reserved_slot;
3307 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
3308 	int err;
3309 
3310 	/* Protects use of hba->reserved_slot. */
3311 	lockdep_assert_held(&hba->dev_cmd.lock);
3312 
3313 	err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
3314 	if (unlikely(err))
3315 		return err;
3316 
3317 	return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout);
3318 }
3319 
3320 /**
3321  * ufshcd_init_query() - init the query response and request parameters
3322  * @hba: per-adapter instance
3323  * @request: address of the request pointer to be initialized
3324  * @response: address of the response pointer to be initialized
3325  * @opcode: operation to perform
3326  * @idn: flag idn to access
3327  * @index: LU number to access
3328  * @selector: query/flag/descriptor further identification
3329  */
ufshcd_init_query(struct ufs_hba * hba,struct ufs_query_req ** request,struct ufs_query_res ** response,enum query_opcode opcode,u8 idn,u8 index,u8 selector)3330 static inline void ufshcd_init_query(struct ufs_hba *hba,
3331 		struct ufs_query_req **request, struct ufs_query_res **response,
3332 		enum query_opcode opcode, u8 idn, u8 index, u8 selector)
3333 {
3334 	*request = &hba->dev_cmd.query.request;
3335 	*response = &hba->dev_cmd.query.response;
3336 	memset(*request, 0, sizeof(struct ufs_query_req));
3337 	memset(*response, 0, sizeof(struct ufs_query_res));
3338 	(*request)->upiu_req.opcode = opcode;
3339 	(*request)->upiu_req.idn = idn;
3340 	(*request)->upiu_req.index = index;
3341 	(*request)->upiu_req.selector = selector;
3342 }
3343 
ufshcd_query_flag_retry(struct ufs_hba * hba,enum query_opcode opcode,enum flag_idn idn,u8 index,bool * flag_res)3344 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
3345 	enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
3346 {
3347 	int ret;
3348 	int retries;
3349 
3350 	for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
3351 		ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
3352 		if (ret)
3353 			dev_dbg(hba->dev,
3354 				"%s: failed with error %d, retries %d\n",
3355 				__func__, ret, retries);
3356 		else
3357 			break;
3358 	}
3359 
3360 	if (ret)
3361 		dev_err(hba->dev,
3362 			"%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n",
3363 			__func__, opcode, idn, ret, retries);
3364 	return ret;
3365 }
3366 
3367 /**
3368  * ufshcd_query_flag() - API function for sending flag query requests
3369  * @hba: per-adapter instance
3370  * @opcode: flag query to perform
3371  * @idn: flag idn to access
3372  * @index: flag index to access
3373  * @flag_res: the flag value after the query request completes
3374  *
3375  * Return: 0 for success, non-zero in case of failure.
3376  */
ufshcd_query_flag(struct ufs_hba * hba,enum query_opcode opcode,enum flag_idn idn,u8 index,bool * flag_res)3377 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
3378 			enum flag_idn idn, u8 index, bool *flag_res)
3379 {
3380 	struct ufs_query_req *request = NULL;
3381 	struct ufs_query_res *response = NULL;
3382 	int err, selector = 0;
3383 	int timeout = QUERY_REQ_TIMEOUT;
3384 
3385 	BUG_ON(!hba);
3386 
3387 	ufshcd_dev_man_lock(hba);
3388 
3389 	ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3390 			selector);
3391 
3392 	switch (opcode) {
3393 	case UPIU_QUERY_OPCODE_SET_FLAG:
3394 	case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3395 	case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3396 		request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3397 		break;
3398 	case UPIU_QUERY_OPCODE_READ_FLAG:
3399 		request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3400 		if (!flag_res) {
3401 			/* No dummy reads */
3402 			dev_err(hba->dev, "%s: Invalid argument for read request\n",
3403 					__func__);
3404 			err = -EINVAL;
3405 			goto out_unlock;
3406 		}
3407 		break;
3408 	default:
3409 		dev_err(hba->dev,
3410 			"%s: Expected query flag opcode but got = %d\n",
3411 			__func__, opcode);
3412 		err = -EINVAL;
3413 		goto out_unlock;
3414 	}
3415 
3416 	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
3417 
3418 	if (err) {
3419 		dev_err(hba->dev,
3420 			"%s: Sending flag query for idn %d failed, err = %d\n",
3421 			__func__, idn, err);
3422 		goto out_unlock;
3423 	}
3424 
3425 	if (flag_res)
3426 		*flag_res = (be32_to_cpu(response->upiu_res.value) &
3427 				MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3428 
3429 out_unlock:
3430 	ufshcd_dev_man_unlock(hba);
3431 	return err;
3432 }
3433 
3434 /**
3435  * ufshcd_query_attr - API function for sending attribute requests
3436  * @hba: per-adapter instance
3437  * @opcode: attribute opcode
3438  * @idn: attribute idn to access
3439  * @index: index field
3440  * @selector: selector field
3441  * @attr_val: the attribute value after the query request completes
3442  *
3443  * Return: 0 for success, non-zero in case of failure.
3444 */
ufshcd_query_attr(struct ufs_hba * hba,enum query_opcode opcode,enum attr_idn idn,u8 index,u8 selector,u32 * attr_val)3445 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3446 		      enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
3447 {
3448 	struct ufs_query_req *request = NULL;
3449 	struct ufs_query_res *response = NULL;
3450 	int err;
3451 
3452 	BUG_ON(!hba);
3453 
3454 	if (!attr_val) {
3455 		dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3456 				__func__, opcode);
3457 		return -EINVAL;
3458 	}
3459 
3460 	ufshcd_dev_man_lock(hba);
3461 
3462 	ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3463 			selector);
3464 
3465 	switch (opcode) {
3466 	case UPIU_QUERY_OPCODE_WRITE_ATTR:
3467 		request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3468 		request->upiu_req.value = cpu_to_be32(*attr_val);
3469 		break;
3470 	case UPIU_QUERY_OPCODE_READ_ATTR:
3471 		request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3472 		break;
3473 	default:
3474 		dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3475 				__func__, opcode);
3476 		err = -EINVAL;
3477 		goto out_unlock;
3478 	}
3479 
3480 	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3481 
3482 	if (err) {
3483 		dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3484 				__func__, opcode, idn, index, err);
3485 		goto out_unlock;
3486 	}
3487 
3488 	*attr_val = be32_to_cpu(response->upiu_res.value);
3489 
3490 out_unlock:
3491 	ufshcd_dev_man_unlock(hba);
3492 	return err;
3493 }
3494 
3495 /**
3496  * ufshcd_query_attr_retry() - API function for sending query
3497  * attribute with retries
3498  * @hba: per-adapter instance
3499  * @opcode: attribute opcode
3500  * @idn: attribute idn to access
3501  * @index: index field
3502  * @selector: selector field
3503  * @attr_val: the attribute value after the query request
3504  * completes
3505  *
3506  * Return: 0 for success, non-zero in case of failure.
3507 */
ufshcd_query_attr_retry(struct ufs_hba * hba,enum query_opcode opcode,enum attr_idn idn,u8 index,u8 selector,u32 * attr_val)3508 int ufshcd_query_attr_retry(struct ufs_hba *hba,
3509 	enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3510 	u32 *attr_val)
3511 {
3512 	int ret = 0;
3513 	u32 retries;
3514 
3515 	for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3516 		ret = ufshcd_query_attr(hba, opcode, idn, index,
3517 						selector, attr_val);
3518 		if (ret)
3519 			dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3520 				__func__, ret, retries);
3521 		else
3522 			break;
3523 	}
3524 
3525 	if (ret)
3526 		dev_err(hba->dev,
3527 			"%s: query attribute, idn %d, failed with error %d after %d retries\n",
3528 			__func__, idn, ret, QUERY_REQ_RETRIES);
3529 	return ret;
3530 }
3531 
__ufshcd_query_descriptor(struct ufs_hba * hba,enum query_opcode opcode,enum desc_idn idn,u8 index,u8 selector,u8 * desc_buf,int * buf_len)3532 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3533 			enum query_opcode opcode, enum desc_idn idn, u8 index,
3534 			u8 selector, u8 *desc_buf, int *buf_len)
3535 {
3536 	struct ufs_query_req *request = NULL;
3537 	struct ufs_query_res *response = NULL;
3538 	int err;
3539 
3540 	BUG_ON(!hba);
3541 
3542 	if (!desc_buf) {
3543 		dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3544 				__func__, opcode);
3545 		return -EINVAL;
3546 	}
3547 
3548 	if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3549 		dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3550 				__func__, *buf_len);
3551 		return -EINVAL;
3552 	}
3553 
3554 	ufshcd_dev_man_lock(hba);
3555 
3556 	ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3557 			selector);
3558 	hba->dev_cmd.query.descriptor = desc_buf;
3559 	request->upiu_req.length = cpu_to_be16(*buf_len);
3560 
3561 	switch (opcode) {
3562 	case UPIU_QUERY_OPCODE_WRITE_DESC:
3563 		request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3564 		break;
3565 	case UPIU_QUERY_OPCODE_READ_DESC:
3566 		request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3567 		break;
3568 	default:
3569 		dev_err(hba->dev,
3570 				"%s: Expected query descriptor opcode but got = 0x%.2x\n",
3571 				__func__, opcode);
3572 		err = -EINVAL;
3573 		goto out_unlock;
3574 	}
3575 
3576 	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3577 
3578 	if (err) {
3579 		dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3580 				__func__, opcode, idn, index, err);
3581 		goto out_unlock;
3582 	}
3583 
3584 	*buf_len = be16_to_cpu(response->upiu_res.length);
3585 
3586 out_unlock:
3587 	hba->dev_cmd.query.descriptor = NULL;
3588 	ufshcd_dev_man_unlock(hba);
3589 	return err;
3590 }
3591 
3592 /**
3593  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3594  * @hba: per-adapter instance
3595  * @opcode: attribute opcode
3596  * @idn: attribute idn to access
3597  * @index: index field
3598  * @selector: selector field
3599  * @desc_buf: the buffer that contains the descriptor
3600  * @buf_len: length parameter passed to the device
3601  *
3602  * The buf_len parameter will contain, on return, the length parameter
3603  * received on the response.
3604  *
3605  * Return: 0 for success, non-zero in case of failure.
3606  */
ufshcd_query_descriptor_retry(struct ufs_hba * hba,enum query_opcode opcode,enum desc_idn idn,u8 index,u8 selector,u8 * desc_buf,int * buf_len)3607 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3608 				  enum query_opcode opcode,
3609 				  enum desc_idn idn, u8 index,
3610 				  u8 selector,
3611 				  u8 *desc_buf, int *buf_len)
3612 {
3613 	int err;
3614 	int retries;
3615 
3616 	for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3617 		err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3618 						selector, desc_buf, buf_len);
3619 		if (!err || err == -EINVAL)
3620 			break;
3621 	}
3622 
3623 	return err;
3624 }
3625 
3626 /**
3627  * ufshcd_read_desc_param - read the specified descriptor parameter
3628  * @hba: Pointer to adapter instance
3629  * @desc_id: descriptor idn value
3630  * @desc_index: descriptor index
3631  * @param_offset: offset of the parameter to read
3632  * @param_read_buf: pointer to buffer where parameter would be read
3633  * @param_size: sizeof(param_read_buf)
3634  *
3635  * Return: 0 in case of success, non-zero otherwise.
3636  */
ufshcd_read_desc_param(struct ufs_hba * hba,enum desc_idn desc_id,int desc_index,u8 param_offset,u8 * param_read_buf,u8 param_size)3637 int ufshcd_read_desc_param(struct ufs_hba *hba,
3638 			   enum desc_idn desc_id,
3639 			   int desc_index,
3640 			   u8 param_offset,
3641 			   u8 *param_read_buf,
3642 			   u8 param_size)
3643 {
3644 	int ret;
3645 	u8 *desc_buf;
3646 	int buff_len = QUERY_DESC_MAX_SIZE;
3647 	bool is_kmalloc = true;
3648 
3649 	/* Safety check */
3650 	if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3651 		return -EINVAL;
3652 
3653 	/* Check whether we need temp memory */
3654 	if (param_offset != 0 || param_size < buff_len) {
3655 		desc_buf = kzalloc(buff_len, GFP_KERNEL);
3656 		if (!desc_buf)
3657 			return -ENOMEM;
3658 	} else {
3659 		desc_buf = param_read_buf;
3660 		is_kmalloc = false;
3661 	}
3662 
3663 	/* Request for full descriptor */
3664 	ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3665 					    desc_id, desc_index, 0,
3666 					    desc_buf, &buff_len);
3667 	if (ret) {
3668 		dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3669 			__func__, desc_id, desc_index, param_offset, ret);
3670 		goto out;
3671 	}
3672 
3673 	/* Update descriptor length */
3674 	buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3675 
3676 	if (param_offset >= buff_len) {
3677 		dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3678 			__func__, param_offset, desc_id, buff_len);
3679 		ret = -EINVAL;
3680 		goto out;
3681 	}
3682 
3683 	/* Sanity check */
3684 	if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3685 		dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3686 			__func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3687 		ret = -EINVAL;
3688 		goto out;
3689 	}
3690 
3691 	if (is_kmalloc) {
3692 		/* Make sure we don't copy more data than available */
3693 		if (param_offset >= buff_len)
3694 			ret = -EINVAL;
3695 		else
3696 			memcpy(param_read_buf, &desc_buf[param_offset],
3697 			       min_t(u32, param_size, buff_len - param_offset));
3698 	}
3699 out:
3700 	if (is_kmalloc)
3701 		kfree(desc_buf);
3702 	return ret;
3703 }
3704 
3705 /**
3706  * struct uc_string_id - unicode string
3707  *
3708  * @len: size of this descriptor inclusive
3709  * @type: descriptor type
3710  * @uc: unicode string character
3711  */
3712 struct uc_string_id {
3713 	u8 len;
3714 	u8 type;
3715 	wchar_t uc[];
3716 } __packed;
3717 
3718 /* replace non-printable or non-ASCII characters with spaces */
ufshcd_remove_non_printable(u8 ch)3719 static inline char ufshcd_remove_non_printable(u8 ch)
3720 {
3721 	return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3722 }
3723 
3724 /**
3725  * ufshcd_read_string_desc - read string descriptor
3726  * @hba: pointer to adapter instance
3727  * @desc_index: descriptor index
3728  * @buf: pointer to buffer where descriptor would be read,
3729  *       the caller should free the memory.
3730  * @ascii: if true convert from unicode to ascii characters
3731  *         null terminated string.
3732  *
3733  * Return:
3734  * *      string size on success.
3735  * *      -ENOMEM: on allocation failure
3736  * *      -EINVAL: on a wrong parameter
3737  */
ufshcd_read_string_desc(struct ufs_hba * hba,u8 desc_index,u8 ** buf,bool ascii)3738 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3739 			    u8 **buf, bool ascii)
3740 {
3741 	struct uc_string_id *uc_str;
3742 	u8 *str;
3743 	int ret;
3744 
3745 	if (!buf)
3746 		return -EINVAL;
3747 
3748 	uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3749 	if (!uc_str)
3750 		return -ENOMEM;
3751 
3752 	ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3753 				     (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3754 	if (ret < 0) {
3755 		dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3756 			QUERY_REQ_RETRIES, ret);
3757 		str = NULL;
3758 		goto out;
3759 	}
3760 
3761 	if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3762 		dev_dbg(hba->dev, "String Desc is of zero length\n");
3763 		str = NULL;
3764 		ret = 0;
3765 		goto out;
3766 	}
3767 
3768 	if (ascii) {
3769 		ssize_t ascii_len;
3770 		int i;
3771 		/* remove header and divide by 2 to move from UTF16 to UTF8 */
3772 		ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3773 		str = kzalloc(ascii_len, GFP_KERNEL);
3774 		if (!str) {
3775 			ret = -ENOMEM;
3776 			goto out;
3777 		}
3778 
3779 		/*
3780 		 * the descriptor contains string in UTF16 format
3781 		 * we need to convert to utf-8 so it can be displayed
3782 		 */
3783 		ret = utf16s_to_utf8s(uc_str->uc,
3784 				      uc_str->len - QUERY_DESC_HDR_SIZE,
3785 				      UTF16_BIG_ENDIAN, str, ascii_len - 1);
3786 
3787 		/* replace non-printable or non-ASCII characters with spaces */
3788 		for (i = 0; i < ret; i++)
3789 			str[i] = ufshcd_remove_non_printable(str[i]);
3790 
3791 		str[ret++] = '\0';
3792 
3793 	} else {
3794 		str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3795 		if (!str) {
3796 			ret = -ENOMEM;
3797 			goto out;
3798 		}
3799 		ret = uc_str->len;
3800 	}
3801 out:
3802 	*buf = str;
3803 	kfree(uc_str);
3804 	return ret;
3805 }
3806 
3807 /**
3808  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3809  * @hba: Pointer to adapter instance
3810  * @lun: lun id
3811  * @param_offset: offset of the parameter to read
3812  * @param_read_buf: pointer to buffer where parameter would be read
3813  * @param_size: sizeof(param_read_buf)
3814  *
3815  * Return: 0 in case of success, non-zero otherwise.
3816  */
ufshcd_read_unit_desc_param(struct ufs_hba * hba,int lun,enum unit_desc_param param_offset,u8 * param_read_buf,u32 param_size)3817 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3818 					      int lun,
3819 					      enum unit_desc_param param_offset,
3820 					      u8 *param_read_buf,
3821 					      u32 param_size)
3822 {
3823 	/*
3824 	 * Unit descriptors are only available for general purpose LUs (LUN id
3825 	 * from 0 to 7) and RPMB Well known LU.
3826 	 */
3827 	if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
3828 		return -EOPNOTSUPP;
3829 
3830 	return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3831 				      param_offset, param_read_buf, param_size);
3832 }
3833 
ufshcd_get_ref_clk_gating_wait(struct ufs_hba * hba)3834 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3835 {
3836 	int err = 0;
3837 	u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3838 
3839 	if (hba->dev_info.wspecversion >= 0x300) {
3840 		err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3841 				QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3842 				&gating_wait);
3843 		if (err)
3844 			dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3845 					 err, gating_wait);
3846 
3847 		if (gating_wait == 0) {
3848 			gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3849 			dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3850 					 gating_wait);
3851 		}
3852 
3853 		hba->dev_info.clk_gating_wait_us = gating_wait;
3854 	}
3855 
3856 	return err;
3857 }
3858 
3859 /**
3860  * ufshcd_memory_alloc - allocate memory for host memory space data structures
3861  * @hba: per adapter instance
3862  *
3863  * 1. Allocate DMA memory for Command Descriptor array
3864  *	Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3865  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3866  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3867  *	(UTMRDL)
3868  * 4. Allocate memory for local reference block(lrb).
3869  *
3870  * Return: 0 for success, non-zero in case of failure.
3871  */
ufshcd_memory_alloc(struct ufs_hba * hba)3872 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3873 {
3874 	size_t utmrdl_size, utrdl_size, ucdl_size;
3875 
3876 	/* Allocate memory for UTP command descriptors */
3877 	ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs;
3878 	hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3879 						  ucdl_size,
3880 						  &hba->ucdl_dma_addr,
3881 						  GFP_KERNEL);
3882 
3883 	/*
3884 	 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3885 	 */
3886 	if (!hba->ucdl_base_addr ||
3887 	    WARN_ON(hba->ucdl_dma_addr & (128 - 1))) {
3888 		dev_err(hba->dev,
3889 			"Command Descriptor Memory allocation failed\n");
3890 		goto out;
3891 	}
3892 
3893 	/*
3894 	 * Allocate memory for UTP Transfer descriptors
3895 	 * UFSHCI requires 1KB alignment of UTRD
3896 	 */
3897 	utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3898 	hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3899 						   utrdl_size,
3900 						   &hba->utrdl_dma_addr,
3901 						   GFP_KERNEL);
3902 	if (!hba->utrdl_base_addr ||
3903 	    WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) {
3904 		dev_err(hba->dev,
3905 			"Transfer Descriptor Memory allocation failed\n");
3906 		goto out;
3907 	}
3908 
3909 	/*
3910 	 * Skip utmrdl allocation; it may have been
3911 	 * allocated during first pass and not released during
3912 	 * MCQ memory allocation.
3913 	 * See ufshcd_release_sdb_queue() and ufshcd_config_mcq()
3914 	 */
3915 	if (hba->utmrdl_base_addr)
3916 		goto skip_utmrdl;
3917 	/*
3918 	 * Allocate memory for UTP Task Management descriptors
3919 	 * UFSHCI requires 1KB alignment of UTMRD
3920 	 */
3921 	utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3922 	hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3923 						    utmrdl_size,
3924 						    &hba->utmrdl_dma_addr,
3925 						    GFP_KERNEL);
3926 	if (!hba->utmrdl_base_addr ||
3927 	    WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) {
3928 		dev_err(hba->dev,
3929 		"Task Management Descriptor Memory allocation failed\n");
3930 		goto out;
3931 	}
3932 
3933 skip_utmrdl:
3934 	/* Allocate memory for local reference block */
3935 	hba->lrb = devm_kcalloc(hba->dev,
3936 				hba->nutrs, sizeof(struct ufshcd_lrb),
3937 				GFP_KERNEL);
3938 	if (!hba->lrb) {
3939 		dev_err(hba->dev, "LRB Memory allocation failed\n");
3940 		goto out;
3941 	}
3942 	return 0;
3943 out:
3944 	return -ENOMEM;
3945 }
3946 
3947 /**
3948  * ufshcd_host_memory_configure - configure local reference block with
3949  *				memory offsets
3950  * @hba: per adapter instance
3951  *
3952  * Configure Host memory space
3953  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3954  * address.
3955  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3956  * and PRDT offset.
3957  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3958  * into local reference block.
3959  */
ufshcd_host_memory_configure(struct ufs_hba * hba)3960 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3961 {
3962 	struct utp_transfer_req_desc *utrdlp;
3963 	dma_addr_t cmd_desc_dma_addr;
3964 	dma_addr_t cmd_desc_element_addr;
3965 	u16 response_offset;
3966 	u16 prdt_offset;
3967 	int cmd_desc_size;
3968 	int i;
3969 
3970 	utrdlp = hba->utrdl_base_addr;
3971 
3972 	response_offset =
3973 		offsetof(struct utp_transfer_cmd_desc, response_upiu);
3974 	prdt_offset =
3975 		offsetof(struct utp_transfer_cmd_desc, prd_table);
3976 
3977 	cmd_desc_size = ufshcd_get_ucd_size(hba);
3978 	cmd_desc_dma_addr = hba->ucdl_dma_addr;
3979 
3980 	for (i = 0; i < hba->nutrs; i++) {
3981 		/* Configure UTRD with command descriptor base address */
3982 		cmd_desc_element_addr =
3983 				(cmd_desc_dma_addr + (cmd_desc_size * i));
3984 		utrdlp[i].command_desc_base_addr =
3985 				cpu_to_le64(cmd_desc_element_addr);
3986 
3987 		/* Response upiu and prdt offset should be in double words */
3988 		if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3989 			utrdlp[i].response_upiu_offset =
3990 				cpu_to_le16(response_offset);
3991 			utrdlp[i].prd_table_offset =
3992 				cpu_to_le16(prdt_offset);
3993 			utrdlp[i].response_upiu_length =
3994 				cpu_to_le16(ALIGNED_UPIU_SIZE);
3995 		} else {
3996 			utrdlp[i].response_upiu_offset =
3997 				cpu_to_le16(response_offset >> 2);
3998 			utrdlp[i].prd_table_offset =
3999 				cpu_to_le16(prdt_offset >> 2);
4000 			utrdlp[i].response_upiu_length =
4001 				cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
4002 		}
4003 
4004 		ufshcd_init_lrb(hba, &hba->lrb[i], i);
4005 	}
4006 }
4007 
4008 /**
4009  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
4010  * @hba: per adapter instance
4011  *
4012  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
4013  * in order to initialize the Unipro link startup procedure.
4014  * Once the Unipro links are up, the device connected to the controller
4015  * is detected.
4016  *
4017  * Return: 0 on success, non-zero value on failure.
4018  */
ufshcd_dme_link_startup(struct ufs_hba * hba)4019 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
4020 {
4021 	struct uic_command uic_cmd = {
4022 		.command = UIC_CMD_DME_LINK_STARTUP,
4023 	};
4024 	int ret;
4025 
4026 	ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4027 	if (ret)
4028 		dev_dbg(hba->dev,
4029 			"dme-link-startup: error code %d\n", ret);
4030 	return ret;
4031 }
4032 /**
4033  * ufshcd_dme_reset - UIC command for DME_RESET
4034  * @hba: per adapter instance
4035  *
4036  * DME_RESET command is issued in order to reset UniPro stack.
4037  * This function now deals with cold reset.
4038  *
4039  * Return: 0 on success, non-zero value on failure.
4040  */
ufshcd_dme_reset(struct ufs_hba * hba)4041 static int ufshcd_dme_reset(struct ufs_hba *hba)
4042 {
4043 	struct uic_command uic_cmd = {
4044 		.command = UIC_CMD_DME_RESET,
4045 	};
4046 	int ret;
4047 
4048 	ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4049 	if (ret)
4050 		dev_err(hba->dev,
4051 			"dme-reset: error code %d\n", ret);
4052 
4053 	return ret;
4054 }
4055 
ufshcd_dme_configure_adapt(struct ufs_hba * hba,int agreed_gear,int adapt_val)4056 int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
4057 			       int agreed_gear,
4058 			       int adapt_val)
4059 {
4060 	int ret;
4061 
4062 	if (agreed_gear < UFS_HS_G4)
4063 		adapt_val = PA_NO_ADAPT;
4064 
4065 	ret = ufshcd_dme_set(hba,
4066 			     UIC_ARG_MIB(PA_TXHSADAPTTYPE),
4067 			     adapt_val);
4068 	return ret;
4069 }
4070 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
4071 
4072 /**
4073  * ufshcd_dme_enable - UIC command for DME_ENABLE
4074  * @hba: per adapter instance
4075  *
4076  * DME_ENABLE command is issued in order to enable UniPro stack.
4077  *
4078  * Return: 0 on success, non-zero value on failure.
4079  */
ufshcd_dme_enable(struct ufs_hba * hba)4080 static int ufshcd_dme_enable(struct ufs_hba *hba)
4081 {
4082 	struct uic_command uic_cmd = {
4083 		.command = UIC_CMD_DME_ENABLE,
4084 	};
4085 	int ret;
4086 
4087 	ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4088 	if (ret)
4089 		dev_err(hba->dev,
4090 			"dme-enable: error code %d\n", ret);
4091 
4092 	return ret;
4093 }
4094 
ufshcd_add_delay_before_dme_cmd(struct ufs_hba * hba)4095 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
4096 {
4097 	#define MIN_DELAY_BEFORE_DME_CMDS_US	1000
4098 	unsigned long min_sleep_time_us;
4099 
4100 	if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
4101 		return;
4102 
4103 	/*
4104 	 * last_dme_cmd_tstamp will be 0 only for 1st call to
4105 	 * this function
4106 	 */
4107 	if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
4108 		min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
4109 	} else {
4110 		unsigned long delta =
4111 			(unsigned long) ktime_to_us(
4112 				ktime_sub(ktime_get(),
4113 				hba->last_dme_cmd_tstamp));
4114 
4115 		if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
4116 			min_sleep_time_us =
4117 				MIN_DELAY_BEFORE_DME_CMDS_US - delta;
4118 		else
4119 			min_sleep_time_us = 0; /* no more delay required */
4120 	}
4121 
4122 	if (min_sleep_time_us > 0) {
4123 		/* allow sleep for extra 50us if needed */
4124 		usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
4125 	}
4126 
4127 	/* update the last_dme_cmd_tstamp */
4128 	hba->last_dme_cmd_tstamp = ktime_get();
4129 }
4130 
4131 /**
4132  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
4133  * @hba: per adapter instance
4134  * @attr_sel: uic command argument1
4135  * @attr_set: attribute set type as uic command argument2
4136  * @mib_val: setting value as uic command argument3
4137  * @peer: indicate whether peer or local
4138  *
4139  * Return: 0 on success, non-zero value on failure.
4140  */
ufshcd_dme_set_attr(struct ufs_hba * hba,u32 attr_sel,u8 attr_set,u32 mib_val,u8 peer)4141 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
4142 			u8 attr_set, u32 mib_val, u8 peer)
4143 {
4144 	struct uic_command uic_cmd = {
4145 		.command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET,
4146 		.argument1 = attr_sel,
4147 		.argument2 = UIC_ARG_ATTR_TYPE(attr_set),
4148 		.argument3 = mib_val,
4149 	};
4150 	static const char *const action[] = {
4151 		"dme-set",
4152 		"dme-peer-set"
4153 	};
4154 	const char *set = action[!!peer];
4155 	int ret;
4156 	int retries = UFS_UIC_COMMAND_RETRIES;
4157 
4158 	do {
4159 		/* for peer attributes we retry upon failure */
4160 		ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4161 		if (ret)
4162 			dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
4163 				set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
4164 	} while (ret && peer && --retries);
4165 
4166 	if (ret)
4167 		dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
4168 			set, UIC_GET_ATTR_ID(attr_sel), mib_val,
4169 			UFS_UIC_COMMAND_RETRIES - retries);
4170 
4171 	return ret;
4172 }
4173 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
4174 
4175 /**
4176  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
4177  * @hba: per adapter instance
4178  * @attr_sel: uic command argument1
4179  * @mib_val: the value of the attribute as returned by the UIC command
4180  * @peer: indicate whether peer or local
4181  *
4182  * Return: 0 on success, non-zero value on failure.
4183  */
ufshcd_dme_get_attr(struct ufs_hba * hba,u32 attr_sel,u32 * mib_val,u8 peer)4184 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
4185 			u32 *mib_val, u8 peer)
4186 {
4187 	struct uic_command uic_cmd = {
4188 		.command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET,
4189 		.argument1 = attr_sel,
4190 	};
4191 	static const char *const action[] = {
4192 		"dme-get",
4193 		"dme-peer-get"
4194 	};
4195 	const char *get = action[!!peer];
4196 	int ret;
4197 	int retries = UFS_UIC_COMMAND_RETRIES;
4198 	struct ufs_pa_layer_attr orig_pwr_info;
4199 	struct ufs_pa_layer_attr temp_pwr_info;
4200 	bool pwr_mode_change = false;
4201 
4202 	if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
4203 		orig_pwr_info = hba->pwr_info;
4204 		temp_pwr_info = orig_pwr_info;
4205 
4206 		if (orig_pwr_info.pwr_tx == FAST_MODE ||
4207 		    orig_pwr_info.pwr_rx == FAST_MODE) {
4208 			temp_pwr_info.pwr_tx = FASTAUTO_MODE;
4209 			temp_pwr_info.pwr_rx = FASTAUTO_MODE;
4210 			pwr_mode_change = true;
4211 		} else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
4212 		    orig_pwr_info.pwr_rx == SLOW_MODE) {
4213 			temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
4214 			temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
4215 			pwr_mode_change = true;
4216 		}
4217 		if (pwr_mode_change) {
4218 			ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
4219 			if (ret)
4220 				goto out;
4221 		}
4222 	}
4223 
4224 	do {
4225 		/* for peer attributes we retry upon failure */
4226 		ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4227 		if (ret)
4228 			dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
4229 				get, UIC_GET_ATTR_ID(attr_sel), ret);
4230 	} while (ret && peer && --retries);
4231 
4232 	if (ret)
4233 		dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
4234 			get, UIC_GET_ATTR_ID(attr_sel),
4235 			UFS_UIC_COMMAND_RETRIES - retries);
4236 
4237 	if (mib_val && !ret)
4238 		*mib_val = uic_cmd.argument3;
4239 
4240 	if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
4241 	    && pwr_mode_change)
4242 		ufshcd_change_power_mode(hba, &orig_pwr_info);
4243 out:
4244 	return ret;
4245 }
4246 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
4247 
4248 /**
4249  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
4250  * state) and waits for it to take effect.
4251  *
4252  * @hba: per adapter instance
4253  * @cmd: UIC command to execute
4254  *
4255  * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
4256  * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
4257  * and device UniPro link and hence it's final completion would be indicated by
4258  * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
4259  * addition to normal UIC command completion Status (UCCS). This function only
4260  * returns after the relevant status bits indicate the completion.
4261  *
4262  * Return: 0 on success, non-zero value on failure.
4263  */
ufshcd_uic_pwr_ctrl(struct ufs_hba * hba,struct uic_command * cmd)4264 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
4265 {
4266 	DECLARE_COMPLETION_ONSTACK(uic_async_done);
4267 	unsigned long flags;
4268 	u8 status;
4269 	int ret;
4270 	bool reenable_intr = false;
4271 
4272 	mutex_lock(&hba->uic_cmd_mutex);
4273 	ufshcd_add_delay_before_dme_cmd(hba);
4274 
4275 	spin_lock_irqsave(hba->host->host_lock, flags);
4276 	if (ufshcd_is_link_broken(hba)) {
4277 		ret = -ENOLINK;
4278 		goto out_unlock;
4279 	}
4280 	hba->uic_async_done = &uic_async_done;
4281 	if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
4282 		ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
4283 		/*
4284 		 * Make sure UIC command completion interrupt is disabled before
4285 		 * issuing UIC command.
4286 		 */
4287 		ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
4288 		reenable_intr = true;
4289 	}
4290 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4291 	ret = __ufshcd_send_uic_cmd(hba, cmd, false);
4292 	if (ret) {
4293 		dev_err(hba->dev,
4294 			"pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
4295 			cmd->command, cmd->argument3, ret);
4296 		goto out;
4297 	}
4298 
4299 	if (!wait_for_completion_timeout(hba->uic_async_done,
4300 					 msecs_to_jiffies(uic_cmd_timeout))) {
4301 		dev_err(hba->dev,
4302 			"pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
4303 			cmd->command, cmd->argument3);
4304 
4305 		if (!cmd->cmd_active) {
4306 			dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
4307 				__func__);
4308 			goto check_upmcrs;
4309 		}
4310 
4311 		ret = -ETIMEDOUT;
4312 		goto out;
4313 	}
4314 
4315 check_upmcrs:
4316 	status = ufshcd_get_upmcrs(hba);
4317 	if (status != PWR_LOCAL) {
4318 		dev_err(hba->dev,
4319 			"pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
4320 			cmd->command, status);
4321 		ret = (status != PWR_OK) ? status : -1;
4322 	}
4323 out:
4324 	if (ret) {
4325 		ufshcd_print_host_state(hba);
4326 		ufshcd_print_pwr_info(hba);
4327 		ufshcd_print_evt_hist(hba);
4328 	}
4329 
4330 	spin_lock_irqsave(hba->host->host_lock, flags);
4331 	hba->active_uic_cmd = NULL;
4332 	hba->uic_async_done = NULL;
4333 	if (reenable_intr)
4334 		ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4335 	if (ret) {
4336 		ufshcd_set_link_broken(hba);
4337 		ufshcd_schedule_eh_work(hba);
4338 	}
4339 out_unlock:
4340 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4341 	mutex_unlock(&hba->uic_cmd_mutex);
4342 
4343 	return ret;
4344 }
4345 
4346 /**
4347  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4348  *				using DME_SET primitives.
4349  * @hba: per adapter instance
4350  * @mode: powr mode value
4351  *
4352  * Return: 0 on success, non-zero value on failure.
4353  */
ufshcd_uic_change_pwr_mode(struct ufs_hba * hba,u8 mode)4354 int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4355 {
4356 	struct uic_command uic_cmd = {
4357 		.command = UIC_CMD_DME_SET,
4358 		.argument1 = UIC_ARG_MIB(PA_PWRMODE),
4359 		.argument3 = mode,
4360 	};
4361 	int ret;
4362 
4363 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4364 		ret = ufshcd_dme_set(hba,
4365 				UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4366 		if (ret) {
4367 			dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4368 						__func__, ret);
4369 			goto out;
4370 		}
4371 	}
4372 
4373 	ufshcd_hold(hba);
4374 	ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4375 	ufshcd_release(hba);
4376 
4377 out:
4378 	return ret;
4379 }
4380 EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode);
4381 
ufshcd_link_recovery(struct ufs_hba * hba)4382 int ufshcd_link_recovery(struct ufs_hba *hba)
4383 {
4384 	int ret;
4385 	unsigned long flags;
4386 
4387 	spin_lock_irqsave(hba->host->host_lock, flags);
4388 	hba->ufshcd_state = UFSHCD_STATE_RESET;
4389 	ufshcd_set_eh_in_progress(hba);
4390 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4391 
4392 	/* Reset the attached device */
4393 	ufshcd_device_reset(hba);
4394 
4395 	ret = ufshcd_host_reset_and_restore(hba);
4396 
4397 	spin_lock_irqsave(hba->host->host_lock, flags);
4398 	if (ret)
4399 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
4400 	ufshcd_clear_eh_in_progress(hba);
4401 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4402 
4403 	if (ret)
4404 		dev_err(hba->dev, "%s: link recovery failed, err %d",
4405 			__func__, ret);
4406 
4407 	return ret;
4408 }
4409 EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
4410 
ufshcd_uic_hibern8_enter(struct ufs_hba * hba)4411 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
4412 {
4413 	struct uic_command uic_cmd = {
4414 		.command = UIC_CMD_DME_HIBER_ENTER,
4415 	};
4416 	ktime_t start = ktime_get();
4417 	int ret;
4418 
4419 	ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4420 
4421 	ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4422 	trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4423 			     ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4424 
4425 	if (ret)
4426 		dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4427 			__func__, ret);
4428 	else
4429 		ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4430 								POST_CHANGE);
4431 
4432 	return ret;
4433 }
4434 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter);
4435 
ufshcd_uic_hibern8_exit(struct ufs_hba * hba)4436 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
4437 {
4438 	struct uic_command uic_cmd = {
4439 		.command = UIC_CMD_DME_HIBER_EXIT,
4440 	};
4441 	int ret;
4442 	ktime_t start = ktime_get();
4443 
4444 	ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4445 
4446 	ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4447 	trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4448 			     ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4449 
4450 	if (ret) {
4451 		dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4452 			__func__, ret);
4453 	} else {
4454 		ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4455 								POST_CHANGE);
4456 		hba->ufs_stats.last_hibern8_exit_tstamp = local_clock();
4457 		hba->ufs_stats.hibern8_exit_cnt++;
4458 	}
4459 
4460 	return ret;
4461 }
4462 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
4463 
ufshcd_configure_auto_hibern8(struct ufs_hba * hba)4464 static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba)
4465 {
4466 	if (!ufshcd_is_auto_hibern8_supported(hba))
4467 		return;
4468 
4469 	ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4470 }
4471 
ufshcd_auto_hibern8_update(struct ufs_hba * hba,u32 ahit)4472 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4473 {
4474 	const u32 cur_ahit = READ_ONCE(hba->ahit);
4475 
4476 	if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit)
4477 		return;
4478 
4479 	WRITE_ONCE(hba->ahit, ahit);
4480 	if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) {
4481 		ufshcd_rpm_get_sync(hba);
4482 		ufshcd_hold(hba);
4483 		ufshcd_configure_auto_hibern8(hba);
4484 		ufshcd_release(hba);
4485 		ufshcd_rpm_put_sync(hba);
4486 	}
4487 }
4488 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4489 
4490  /**
4491  * ufshcd_init_pwr_info - setting the POR (power on reset)
4492  * values in hba power info
4493  * @hba: per-adapter instance
4494  */
ufshcd_init_pwr_info(struct ufs_hba * hba)4495 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4496 {
4497 	hba->pwr_info.gear_rx = UFS_PWM_G1;
4498 	hba->pwr_info.gear_tx = UFS_PWM_G1;
4499 	hba->pwr_info.lane_rx = UFS_LANE_1;
4500 	hba->pwr_info.lane_tx = UFS_LANE_1;
4501 	hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4502 	hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4503 	hba->pwr_info.hs_rate = 0;
4504 }
4505 
4506 /**
4507  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4508  * @hba: per-adapter instance
4509  *
4510  * Return: 0 upon success; < 0 upon failure.
4511  */
ufshcd_get_max_pwr_mode(struct ufs_hba * hba)4512 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4513 {
4514 	struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4515 
4516 	if (hba->max_pwr_info.is_valid)
4517 		return 0;
4518 
4519 	if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) {
4520 		pwr_info->pwr_tx = FASTAUTO_MODE;
4521 		pwr_info->pwr_rx = FASTAUTO_MODE;
4522 	} else {
4523 		pwr_info->pwr_tx = FAST_MODE;
4524 		pwr_info->pwr_rx = FAST_MODE;
4525 	}
4526 	pwr_info->hs_rate = PA_HS_MODE_B;
4527 
4528 	/* Get the connected lane count */
4529 	ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4530 			&pwr_info->lane_rx);
4531 	ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4532 			&pwr_info->lane_tx);
4533 
4534 	if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4535 		dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4536 				__func__,
4537 				pwr_info->lane_rx,
4538 				pwr_info->lane_tx);
4539 		return -EINVAL;
4540 	}
4541 
4542 	/*
4543 	 * First, get the maximum gears of HS speed.
4544 	 * If a zero value, it means there is no HSGEAR capability.
4545 	 * Then, get the maximum gears of PWM speed.
4546 	 */
4547 	ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4548 	if (!pwr_info->gear_rx) {
4549 		ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4550 				&pwr_info->gear_rx);
4551 		if (!pwr_info->gear_rx) {
4552 			dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4553 				__func__, pwr_info->gear_rx);
4554 			return -EINVAL;
4555 		}
4556 		pwr_info->pwr_rx = SLOW_MODE;
4557 	}
4558 
4559 	ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4560 			&pwr_info->gear_tx);
4561 	if (!pwr_info->gear_tx) {
4562 		ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4563 				&pwr_info->gear_tx);
4564 		if (!pwr_info->gear_tx) {
4565 			dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4566 				__func__, pwr_info->gear_tx);
4567 			return -EINVAL;
4568 		}
4569 		pwr_info->pwr_tx = SLOW_MODE;
4570 	}
4571 
4572 	hba->max_pwr_info.is_valid = true;
4573 	return 0;
4574 }
4575 
ufshcd_change_power_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode)4576 static int ufshcd_change_power_mode(struct ufs_hba *hba,
4577 			     struct ufs_pa_layer_attr *pwr_mode)
4578 {
4579 	int ret;
4580 
4581 	/* if already configured to the requested pwr_mode */
4582 	if (!hba->force_pmc &&
4583 	    pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4584 	    pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4585 	    pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4586 	    pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4587 	    pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4588 	    pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4589 	    pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4590 		dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4591 		return 0;
4592 	}
4593 
4594 	/*
4595 	 * Configure attributes for power mode change with below.
4596 	 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4597 	 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4598 	 * - PA_HSSERIES
4599 	 */
4600 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4601 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4602 			pwr_mode->lane_rx);
4603 	if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4604 			pwr_mode->pwr_rx == FAST_MODE)
4605 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true);
4606 	else
4607 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false);
4608 
4609 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4610 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4611 			pwr_mode->lane_tx);
4612 	if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4613 			pwr_mode->pwr_tx == FAST_MODE)
4614 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
4615 	else
4616 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false);
4617 
4618 	if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4619 	    pwr_mode->pwr_tx == FASTAUTO_MODE ||
4620 	    pwr_mode->pwr_rx == FAST_MODE ||
4621 	    pwr_mode->pwr_tx == FAST_MODE)
4622 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4623 						pwr_mode->hs_rate);
4624 
4625 	if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4626 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4627 				DL_FC0ProtectionTimeOutVal_Default);
4628 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4629 				DL_TC0ReplayTimeOutVal_Default);
4630 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4631 				DL_AFC0ReqTimeOutVal_Default);
4632 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4633 				DL_FC1ProtectionTimeOutVal_Default);
4634 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4635 				DL_TC1ReplayTimeOutVal_Default);
4636 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4637 				DL_AFC1ReqTimeOutVal_Default);
4638 
4639 		ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4640 				DL_FC0ProtectionTimeOutVal_Default);
4641 		ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4642 				DL_TC0ReplayTimeOutVal_Default);
4643 		ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4644 				DL_AFC0ReqTimeOutVal_Default);
4645 	}
4646 
4647 	ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4648 			| pwr_mode->pwr_tx);
4649 
4650 	if (ret) {
4651 		dev_err(hba->dev,
4652 			"%s: power mode change failed %d\n", __func__, ret);
4653 	} else {
4654 		ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4655 								pwr_mode);
4656 
4657 		memcpy(&hba->pwr_info, pwr_mode,
4658 			sizeof(struct ufs_pa_layer_attr));
4659 	}
4660 
4661 	return ret;
4662 }
4663 
4664 /**
4665  * ufshcd_config_pwr_mode - configure a new power mode
4666  * @hba: per-adapter instance
4667  * @desired_pwr_mode: desired power configuration
4668  *
4669  * Return: 0 upon success; < 0 upon failure.
4670  */
ufshcd_config_pwr_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * desired_pwr_mode)4671 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4672 		struct ufs_pa_layer_attr *desired_pwr_mode)
4673 {
4674 	struct ufs_pa_layer_attr final_params = { 0 };
4675 	int ret;
4676 
4677 	ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4678 					desired_pwr_mode, &final_params);
4679 
4680 	if (ret)
4681 		memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4682 
4683 	ret = ufshcd_change_power_mode(hba, &final_params);
4684 
4685 	return ret;
4686 }
4687 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4688 
4689 /**
4690  * ufshcd_complete_dev_init() - checks device readiness
4691  * @hba: per-adapter instance
4692  *
4693  * Set fDeviceInit flag and poll until device toggles it.
4694  *
4695  * Return: 0 upon success; < 0 upon failure.
4696  */
ufshcd_complete_dev_init(struct ufs_hba * hba)4697 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4698 {
4699 	int err;
4700 	bool flag_res = true;
4701 	ktime_t timeout;
4702 
4703 	err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4704 		QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4705 	if (err) {
4706 		dev_err(hba->dev,
4707 			"%s: setting fDeviceInit flag failed with error %d\n",
4708 			__func__, err);
4709 		goto out;
4710 	}
4711 
4712 	/* Poll fDeviceInit flag to be cleared */
4713 	timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4714 	do {
4715 		err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4716 					QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4717 		if (!flag_res)
4718 			break;
4719 		usleep_range(500, 1000);
4720 	} while (ktime_before(ktime_get(), timeout));
4721 
4722 	if (err) {
4723 		dev_err(hba->dev,
4724 				"%s: reading fDeviceInit flag failed with error %d\n",
4725 				__func__, err);
4726 	} else if (flag_res) {
4727 		dev_err(hba->dev,
4728 				"%s: fDeviceInit was not cleared by the device\n",
4729 				__func__);
4730 		err = -EBUSY;
4731 	}
4732 out:
4733 	return err;
4734 }
4735 
4736 /**
4737  * ufshcd_make_hba_operational - Make UFS controller operational
4738  * @hba: per adapter instance
4739  *
4740  * To bring UFS host controller to operational state,
4741  * 1. Enable required interrupts
4742  * 2. Configure interrupt aggregation
4743  * 3. Program UTRL and UTMRL base address
4744  * 4. Configure run-stop-registers
4745  *
4746  * Return: 0 on success, non-zero value on failure.
4747  */
ufshcd_make_hba_operational(struct ufs_hba * hba)4748 int ufshcd_make_hba_operational(struct ufs_hba *hba)
4749 {
4750 	int err = 0;
4751 	u32 reg;
4752 
4753 	/* Enable required interrupts */
4754 	ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4755 
4756 	/* Configure interrupt aggregation */
4757 	if (ufshcd_is_intr_aggr_allowed(hba))
4758 		ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4759 	else
4760 		ufshcd_disable_intr_aggr(hba);
4761 
4762 	/* Configure UTRL and UTMRL base address registers */
4763 	ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4764 			REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4765 	ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4766 			REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4767 	ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4768 			REG_UTP_TASK_REQ_LIST_BASE_L);
4769 	ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4770 			REG_UTP_TASK_REQ_LIST_BASE_H);
4771 
4772 	/*
4773 	 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4774 	 */
4775 	reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4776 	if (!(ufshcd_get_lists_status(reg))) {
4777 		ufshcd_enable_run_stop_reg(hba);
4778 	} else {
4779 		dev_err(hba->dev,
4780 			"Host controller not ready to process requests");
4781 		err = -EIO;
4782 	}
4783 
4784 	return err;
4785 }
4786 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4787 
4788 /**
4789  * ufshcd_hba_stop - Send controller to reset state
4790  * @hba: per adapter instance
4791  */
ufshcd_hba_stop(struct ufs_hba * hba)4792 void ufshcd_hba_stop(struct ufs_hba *hba)
4793 {
4794 	unsigned long flags;
4795 	int err;
4796 
4797 	/*
4798 	 * Obtain the host lock to prevent that the controller is disabled
4799 	 * while the UFS interrupt handler is active on another CPU.
4800 	 */
4801 	spin_lock_irqsave(hba->host->host_lock, flags);
4802 	ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
4803 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4804 
4805 	err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4806 					CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4807 					10, 1);
4808 	if (err)
4809 		dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4810 }
4811 EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
4812 
4813 /**
4814  * ufshcd_hba_execute_hce - initialize the controller
4815  * @hba: per adapter instance
4816  *
4817  * The controller resets itself and controller firmware initialization
4818  * sequence kicks off. When controller is ready it will set
4819  * the Host Controller Enable bit to 1.
4820  *
4821  * Return: 0 on success, non-zero value on failure.
4822  */
ufshcd_hba_execute_hce(struct ufs_hba * hba)4823 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4824 {
4825 	int retry_outer = 3;
4826 	int retry_inner;
4827 
4828 start:
4829 	if (ufshcd_is_hba_active(hba))
4830 		/* change controller state to "reset state" */
4831 		ufshcd_hba_stop(hba);
4832 
4833 	/* UniPro link is disabled at this point */
4834 	ufshcd_set_link_off(hba);
4835 
4836 	ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4837 
4838 	/* start controller initialization sequence */
4839 	ufshcd_hba_start(hba);
4840 
4841 	/*
4842 	 * To initialize a UFS host controller HCE bit must be set to 1.
4843 	 * During initialization the HCE bit value changes from 1->0->1.
4844 	 * When the host controller completes initialization sequence
4845 	 * it sets the value of HCE bit to 1. The same HCE bit is read back
4846 	 * to check if the controller has completed initialization sequence.
4847 	 * So without this delay the value HCE = 1, set in the previous
4848 	 * instruction might be read back.
4849 	 * This delay can be changed based on the controller.
4850 	 */
4851 	ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4852 
4853 	/* wait for the host controller to complete initialization */
4854 	retry_inner = 50;
4855 	while (!ufshcd_is_hba_active(hba)) {
4856 		if (retry_inner) {
4857 			retry_inner--;
4858 		} else {
4859 			dev_err(hba->dev,
4860 				"Controller enable failed\n");
4861 			if (retry_outer) {
4862 				retry_outer--;
4863 				goto start;
4864 			}
4865 			return -EIO;
4866 		}
4867 		usleep_range(1000, 1100);
4868 	}
4869 
4870 	/* enable UIC related interrupts */
4871 	ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4872 
4873 	ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4874 
4875 	return 0;
4876 }
4877 
ufshcd_hba_enable(struct ufs_hba * hba)4878 int ufshcd_hba_enable(struct ufs_hba *hba)
4879 {
4880 	int ret;
4881 
4882 	if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4883 		ufshcd_set_link_off(hba);
4884 		ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4885 
4886 		/* enable UIC related interrupts */
4887 		ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4888 		ret = ufshcd_dme_reset(hba);
4889 		if (ret) {
4890 			dev_err(hba->dev, "DME_RESET failed\n");
4891 			return ret;
4892 		}
4893 
4894 		ret = ufshcd_dme_enable(hba);
4895 		if (ret) {
4896 			dev_err(hba->dev, "Enabling DME failed\n");
4897 			return ret;
4898 		}
4899 
4900 		ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4901 	} else {
4902 		ret = ufshcd_hba_execute_hce(hba);
4903 	}
4904 
4905 	return ret;
4906 }
4907 EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4908 
ufshcd_disable_tx_lcc(struct ufs_hba * hba,bool peer)4909 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4910 {
4911 	int tx_lanes = 0, i, err = 0;
4912 
4913 	if (!peer)
4914 		ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4915 			       &tx_lanes);
4916 	else
4917 		ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4918 				    &tx_lanes);
4919 	for (i = 0; i < tx_lanes; i++) {
4920 		if (!peer)
4921 			err = ufshcd_dme_set(hba,
4922 				UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4923 					UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4924 					0);
4925 		else
4926 			err = ufshcd_dme_peer_set(hba,
4927 				UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4928 					UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4929 					0);
4930 		if (err) {
4931 			dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4932 				__func__, peer, i, err);
4933 			break;
4934 		}
4935 	}
4936 
4937 	return err;
4938 }
4939 
ufshcd_disable_device_tx_lcc(struct ufs_hba * hba)4940 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4941 {
4942 	return ufshcd_disable_tx_lcc(hba, true);
4943 }
4944 
ufshcd_update_evt_hist(struct ufs_hba * hba,u32 id,u32 val)4945 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
4946 {
4947 	struct ufs_event_hist *e;
4948 
4949 	if (id >= UFS_EVT_CNT)
4950 		return;
4951 
4952 	e = &hba->ufs_stats.event[id];
4953 	e->val[e->pos] = val;
4954 	e->tstamp[e->pos] = local_clock();
4955 	e->cnt += 1;
4956 	e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4957 
4958 	ufshcd_vops_event_notify(hba, id, &val);
4959 }
4960 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
4961 
4962 /**
4963  * ufshcd_link_startup - Initialize unipro link startup
4964  * @hba: per adapter instance
4965  *
4966  * Return: 0 for success, non-zero in case of failure.
4967  */
ufshcd_link_startup(struct ufs_hba * hba)4968 static int ufshcd_link_startup(struct ufs_hba *hba)
4969 {
4970 	int ret;
4971 	int retries = DME_LINKSTARTUP_RETRIES;
4972 	bool link_startup_again = false;
4973 
4974 	/*
4975 	 * If UFS device isn't active then we will have to issue link startup
4976 	 * 2 times to make sure the device state move to active.
4977 	 */
4978 	if (!ufshcd_is_ufs_dev_active(hba))
4979 		link_startup_again = true;
4980 
4981 link_startup:
4982 	do {
4983 		ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4984 
4985 		ret = ufshcd_dme_link_startup(hba);
4986 
4987 		/* check if device is detected by inter-connect layer */
4988 		if (!ret && !ufshcd_is_device_present(hba)) {
4989 			ufshcd_update_evt_hist(hba,
4990 					       UFS_EVT_LINK_STARTUP_FAIL,
4991 					       0);
4992 			dev_err(hba->dev, "%s: Device not present\n", __func__);
4993 			ret = -ENXIO;
4994 			goto out;
4995 		}
4996 
4997 		/*
4998 		 * DME link lost indication is only received when link is up,
4999 		 * but we can't be sure if the link is up until link startup
5000 		 * succeeds. So reset the local Uni-Pro and try again.
5001 		 */
5002 		if (ret && retries && ufshcd_hba_enable(hba)) {
5003 			ufshcd_update_evt_hist(hba,
5004 					       UFS_EVT_LINK_STARTUP_FAIL,
5005 					       (u32)ret);
5006 			goto out;
5007 		}
5008 	} while (ret && retries--);
5009 
5010 	if (ret) {
5011 		/* failed to get the link up... retire */
5012 		ufshcd_update_evt_hist(hba,
5013 				       UFS_EVT_LINK_STARTUP_FAIL,
5014 				       (u32)ret);
5015 		goto out;
5016 	}
5017 
5018 	if (link_startup_again) {
5019 		link_startup_again = false;
5020 		retries = DME_LINKSTARTUP_RETRIES;
5021 		goto link_startup;
5022 	}
5023 
5024 	/* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
5025 	ufshcd_init_pwr_info(hba);
5026 	ufshcd_print_pwr_info(hba);
5027 
5028 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
5029 		ret = ufshcd_disable_device_tx_lcc(hba);
5030 		if (ret)
5031 			goto out;
5032 	}
5033 
5034 	/* Include any host controller configuration via UIC commands */
5035 	ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
5036 	if (ret)
5037 		goto out;
5038 
5039 	/* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
5040 	ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5041 	ret = ufshcd_make_hba_operational(hba);
5042 out:
5043 	if (ret) {
5044 		dev_err(hba->dev, "link startup failed %d\n", ret);
5045 		ufshcd_print_host_state(hba);
5046 		ufshcd_print_pwr_info(hba);
5047 		ufshcd_print_evt_hist(hba);
5048 	}
5049 	return ret;
5050 }
5051 
5052 /**
5053  * ufshcd_verify_dev_init() - Verify device initialization
5054  * @hba: per-adapter instance
5055  *
5056  * Send NOP OUT UPIU and wait for NOP IN response to check whether the
5057  * device Transport Protocol (UTP) layer is ready after a reset.
5058  * If the UTP layer at the device side is not initialized, it may
5059  * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
5060  * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
5061  *
5062  * Return: 0 upon success; < 0 upon failure.
5063  */
ufshcd_verify_dev_init(struct ufs_hba * hba)5064 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
5065 {
5066 	int err = 0;
5067 	int retries;
5068 
5069 	ufshcd_dev_man_lock(hba);
5070 
5071 	for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
5072 		err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
5073 					  hba->nop_out_timeout);
5074 
5075 		if (!err || err == -ETIMEDOUT)
5076 			break;
5077 
5078 		dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
5079 	}
5080 
5081 	ufshcd_dev_man_unlock(hba);
5082 
5083 	if (err)
5084 		dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
5085 	return err;
5086 }
5087 
5088 /**
5089  * ufshcd_setup_links - associate link b/w device wlun and other luns
5090  * @sdev: pointer to SCSI device
5091  * @hba: pointer to ufs hba
5092  */
ufshcd_setup_links(struct ufs_hba * hba,struct scsi_device * sdev)5093 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev)
5094 {
5095 	struct device_link *link;
5096 
5097 	/*
5098 	 * Device wlun is the supplier & rest of the luns are consumers.
5099 	 * This ensures that device wlun suspends after all other luns.
5100 	 */
5101 	if (hba->ufs_device_wlun) {
5102 		link = device_link_add(&sdev->sdev_gendev,
5103 				       &hba->ufs_device_wlun->sdev_gendev,
5104 				       DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE);
5105 		if (!link) {
5106 			dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
5107 				dev_name(&hba->ufs_device_wlun->sdev_gendev));
5108 			return;
5109 		}
5110 		hba->luns_avail--;
5111 		/* Ignore REPORT_LUN wlun probing */
5112 		if (hba->luns_avail == 1) {
5113 			ufshcd_rpm_put(hba);
5114 			return;
5115 		}
5116 	} else {
5117 		/*
5118 		 * Device wlun is probed. The assumption is that WLUNs are
5119 		 * scanned before other LUNs.
5120 		 */
5121 		hba->luns_avail--;
5122 	}
5123 }
5124 
5125 /**
5126  * ufshcd_lu_init - Initialize the relevant parameters of the LU
5127  * @hba: per-adapter instance
5128  * @sdev: pointer to SCSI device
5129  */
ufshcd_lu_init(struct ufs_hba * hba,struct scsi_device * sdev)5130 static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev)
5131 {
5132 	int len = QUERY_DESC_MAX_SIZE;
5133 	u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun);
5134 	u8 lun_qdepth = hba->nutrs;
5135 	u8 *desc_buf;
5136 	int ret;
5137 
5138 	desc_buf = kzalloc(len, GFP_KERNEL);
5139 	if (!desc_buf)
5140 		goto set_qdepth;
5141 
5142 	ret = ufshcd_read_unit_desc_param(hba, lun, 0, desc_buf, len);
5143 	if (ret < 0) {
5144 		if (ret == -EOPNOTSUPP)
5145 			/* If LU doesn't support unit descriptor, its queue depth is set to 1 */
5146 			lun_qdepth = 1;
5147 		kfree(desc_buf);
5148 		goto set_qdepth;
5149 	}
5150 
5151 	if (desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]) {
5152 		/*
5153 		 * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will
5154 		 * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth
5155 		 */
5156 		lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs);
5157 	}
5158 	/*
5159 	 * According to UFS device specification, the write protection mode is only supported by
5160 	 * normal LU, not supported by WLUN.
5161 	 */
5162 	if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported &&
5163 	    !hba->dev_info.is_lu_power_on_wp &&
5164 	    desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP)
5165 		hba->dev_info.is_lu_power_on_wp = true;
5166 
5167 	/* In case of RPMB LU, check if advanced RPMB mode is enabled */
5168 	if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN &&
5169 	    desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4))
5170 		hba->dev_info.b_advanced_rpmb_en = true;
5171 
5172 
5173 	kfree(desc_buf);
5174 set_qdepth:
5175 	/*
5176 	 * For WLUNs that don't support unit descriptor, queue depth is set to 1. For LUs whose
5177 	 * bLUQueueDepth == 0, the queue depth is set to a maximum value that host can queue.
5178 	 */
5179 	dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth);
5180 	scsi_change_queue_depth(sdev, lun_qdepth);
5181 }
5182 
5183 /**
5184  * ufshcd_slave_alloc - handle initial SCSI device configurations
5185  * @sdev: pointer to SCSI device
5186  *
5187  * Return: success.
5188  */
ufshcd_slave_alloc(struct scsi_device * sdev)5189 static int ufshcd_slave_alloc(struct scsi_device *sdev)
5190 {
5191 	struct ufs_hba *hba;
5192 
5193 	hba = shost_priv(sdev->host);
5194 
5195 	/* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
5196 	sdev->use_10_for_ms = 1;
5197 
5198 	/* DBD field should be set to 1 in mode sense(10) */
5199 	sdev->set_dbd_for_ms = 1;
5200 
5201 	/* allow SCSI layer to restart the device in case of errors */
5202 	sdev->allow_restart = 1;
5203 
5204 	/* REPORT SUPPORTED OPERATION CODES is not supported */
5205 	sdev->no_report_opcodes = 1;
5206 
5207 	/* WRITE_SAME command is not supported */
5208 	sdev->no_write_same = 1;
5209 
5210 	ufshcd_lu_init(hba, sdev);
5211 
5212 	ufshcd_setup_links(hba, sdev);
5213 
5214 	return 0;
5215 }
5216 
5217 /**
5218  * ufshcd_change_queue_depth - change queue depth
5219  * @sdev: pointer to SCSI device
5220  * @depth: required depth to set
5221  *
5222  * Change queue depth and make sure the max. limits are not crossed.
5223  *
5224  * Return: new queue depth.
5225  */
ufshcd_change_queue_depth(struct scsi_device * sdev,int depth)5226 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
5227 {
5228 	return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue));
5229 }
5230 
5231 /**
5232  * ufshcd_device_configure - adjust SCSI device configurations
5233  * @sdev: pointer to SCSI device
5234  * @lim: queue limits
5235  *
5236  * Return: 0 (success).
5237  */
ufshcd_device_configure(struct scsi_device * sdev,struct queue_limits * lim)5238 static int ufshcd_device_configure(struct scsi_device *sdev,
5239 		struct queue_limits *lim)
5240 {
5241 	struct ufs_hba *hba = shost_priv(sdev->host);
5242 	struct request_queue *q = sdev->request_queue;
5243 
5244 	lim->dma_pad_mask = PRDT_DATA_BYTE_COUNT_PAD - 1;
5245 
5246 	/*
5247 	 * Block runtime-pm until all consumers are added.
5248 	 * Refer ufshcd_setup_links().
5249 	 */
5250 	if (is_device_wlun(sdev))
5251 		pm_runtime_get_noresume(&sdev->sdev_gendev);
5252 	else if (ufshcd_is_rpm_autosuspend_allowed(hba))
5253 		sdev->rpm_autosuspend = 1;
5254 	/*
5255 	 * Do not print messages during runtime PM to avoid never-ending cycles
5256 	 * of messages written back to storage by user space causing runtime
5257 	 * resume, causing more messages and so on.
5258 	 */
5259 	sdev->silence_suspend = 1;
5260 
5261 	ufshcd_crypto_register(hba, q);
5262 
5263 	return 0;
5264 }
5265 
5266 /**
5267  * ufshcd_slave_destroy - remove SCSI device configurations
5268  * @sdev: pointer to SCSI device
5269  */
ufshcd_slave_destroy(struct scsi_device * sdev)5270 static void ufshcd_slave_destroy(struct scsi_device *sdev)
5271 {
5272 	struct ufs_hba *hba;
5273 	unsigned long flags;
5274 
5275 	hba = shost_priv(sdev->host);
5276 
5277 	/* Drop the reference as it won't be needed anymore */
5278 	if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
5279 		spin_lock_irqsave(hba->host->host_lock, flags);
5280 		hba->ufs_device_wlun = NULL;
5281 		spin_unlock_irqrestore(hba->host->host_lock, flags);
5282 	} else if (hba->ufs_device_wlun) {
5283 		struct device *supplier = NULL;
5284 
5285 		/* Ensure UFS Device WLUN exists and does not disappear */
5286 		spin_lock_irqsave(hba->host->host_lock, flags);
5287 		if (hba->ufs_device_wlun) {
5288 			supplier = &hba->ufs_device_wlun->sdev_gendev;
5289 			get_device(supplier);
5290 		}
5291 		spin_unlock_irqrestore(hba->host->host_lock, flags);
5292 
5293 		if (supplier) {
5294 			/*
5295 			 * If a LUN fails to probe (e.g. absent BOOT WLUN), the
5296 			 * device will not have been registered but can still
5297 			 * have a device link holding a reference to the device.
5298 			 */
5299 			device_link_remove(&sdev->sdev_gendev, supplier);
5300 			put_device(supplier);
5301 		}
5302 	}
5303 }
5304 
5305 /**
5306  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
5307  * @lrbp: pointer to local reference block of completed command
5308  * @scsi_status: SCSI command status
5309  *
5310  * Return: value base on SCSI command status.
5311  */
5312 static inline int
ufshcd_scsi_cmd_status(struct ufshcd_lrb * lrbp,int scsi_status)5313 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
5314 {
5315 	int result = 0;
5316 
5317 	switch (scsi_status) {
5318 	case SAM_STAT_CHECK_CONDITION:
5319 		ufshcd_copy_sense_data(lrbp);
5320 		fallthrough;
5321 	case SAM_STAT_GOOD:
5322 		result |= DID_OK << 16 | scsi_status;
5323 		break;
5324 	case SAM_STAT_TASK_SET_FULL:
5325 	case SAM_STAT_BUSY:
5326 	case SAM_STAT_TASK_ABORTED:
5327 		ufshcd_copy_sense_data(lrbp);
5328 		result |= scsi_status;
5329 		break;
5330 	default:
5331 		result |= DID_ERROR << 16;
5332 		break;
5333 	} /* end of switch */
5334 
5335 	return result;
5336 }
5337 
5338 /**
5339  * ufshcd_transfer_rsp_status - Get overall status of the response
5340  * @hba: per adapter instance
5341  * @lrbp: pointer to local reference block of completed command
5342  * @cqe: pointer to the completion queue entry
5343  *
5344  * Return: result of the command to notify SCSI midlayer.
5345  */
5346 static inline int
ufshcd_transfer_rsp_status(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,struct cq_entry * cqe)5347 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
5348 			   struct cq_entry *cqe)
5349 {
5350 	int result = 0;
5351 	int scsi_status;
5352 	enum utp_ocs ocs;
5353 	u8 upiu_flags;
5354 	u32 resid;
5355 
5356 	upiu_flags = lrbp->ucd_rsp_ptr->header.flags;
5357 	resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count);
5358 	/*
5359 	 * Test !overflow instead of underflow to support UFS devices that do
5360 	 * not set either flag.
5361 	 */
5362 	if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW))
5363 		scsi_set_resid(lrbp->cmd, resid);
5364 
5365 	/* overall command status of utrd */
5366 	ocs = ufshcd_get_tr_ocs(lrbp, cqe);
5367 
5368 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5369 		if (lrbp->ucd_rsp_ptr->header.response ||
5370 		    lrbp->ucd_rsp_ptr->header.status)
5371 			ocs = OCS_SUCCESS;
5372 	}
5373 
5374 	switch (ocs) {
5375 	case OCS_SUCCESS:
5376 		hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5377 		switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) {
5378 		case UPIU_TRANSACTION_RESPONSE:
5379 			/*
5380 			 * get the result based on SCSI status response
5381 			 * to notify the SCSI midlayer of the command status
5382 			 */
5383 			scsi_status = lrbp->ucd_rsp_ptr->header.status;
5384 			result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
5385 
5386 			/*
5387 			 * Currently we are only supporting BKOPs exception
5388 			 * events hence we can ignore BKOPs exception event
5389 			 * during power management callbacks. BKOPs exception
5390 			 * event is not expected to be raised in runtime suspend
5391 			 * callback as it allows the urgent bkops.
5392 			 * During system suspend, we are anyway forcefully
5393 			 * disabling the bkops and if urgent bkops is needed
5394 			 * it will be enabled on system resume. Long term
5395 			 * solution could be to abort the system suspend if
5396 			 * UFS device needs urgent BKOPs.
5397 			 */
5398 			if (!hba->pm_op_in_progress &&
5399 			    !ufshcd_eh_in_progress(hba) &&
5400 			    ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5401 				/* Flushed in suspend */
5402 				schedule_work(&hba->eeh_work);
5403 			break;
5404 		case UPIU_TRANSACTION_REJECT_UPIU:
5405 			/* TODO: handle Reject UPIU Response */
5406 			result = DID_ERROR << 16;
5407 			dev_err(hba->dev,
5408 				"Reject UPIU not fully implemented\n");
5409 			break;
5410 		default:
5411 			dev_err(hba->dev,
5412 				"Unexpected request response code = %x\n",
5413 				result);
5414 			result = DID_ERROR << 16;
5415 			break;
5416 		}
5417 		break;
5418 	case OCS_ABORTED:
5419 	case OCS_INVALID_COMMAND_STATUS:
5420 		result |= DID_REQUEUE << 16;
5421 		dev_warn(hba->dev,
5422 				"OCS %s from controller for tag %d\n",
5423 				(ocs == OCS_ABORTED ? "aborted" : "invalid"),
5424 				lrbp->task_tag);
5425 		break;
5426 	case OCS_INVALID_CMD_TABLE_ATTR:
5427 	case OCS_INVALID_PRDT_ATTR:
5428 	case OCS_MISMATCH_DATA_BUF_SIZE:
5429 	case OCS_MISMATCH_RESP_UPIU_SIZE:
5430 	case OCS_PEER_COMM_FAILURE:
5431 	case OCS_FATAL_ERROR:
5432 	case OCS_DEVICE_FATAL_ERROR:
5433 	case OCS_INVALID_CRYPTO_CONFIG:
5434 	case OCS_GENERAL_CRYPTO_ERROR:
5435 	default:
5436 		result |= DID_ERROR << 16;
5437 		dev_err(hba->dev,
5438 				"OCS error from controller = %x for tag %d\n",
5439 				ocs, lrbp->task_tag);
5440 		ufshcd_print_evt_hist(hba);
5441 		ufshcd_print_host_state(hba);
5442 		break;
5443 	} /* end of switch */
5444 
5445 	if ((host_byte(result) != DID_OK) &&
5446 	    (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5447 		ufshcd_print_tr(hba, lrbp->task_tag, true);
5448 	return result;
5449 }
5450 
ufshcd_is_auto_hibern8_error(struct ufs_hba * hba,u32 intr_mask)5451 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5452 					 u32 intr_mask)
5453 {
5454 	if (!ufshcd_is_auto_hibern8_supported(hba) ||
5455 	    !ufshcd_is_auto_hibern8_enabled(hba))
5456 		return false;
5457 
5458 	if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5459 		return false;
5460 
5461 	if (hba->active_uic_cmd &&
5462 	    (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5463 	    hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5464 		return false;
5465 
5466 	return true;
5467 }
5468 
5469 /**
5470  * ufshcd_uic_cmd_compl - handle completion of uic command
5471  * @hba: per adapter instance
5472  * @intr_status: interrupt status generated by the controller
5473  *
5474  * Return:
5475  *  IRQ_HANDLED - If interrupt is valid
5476  *  IRQ_NONE    - If invalid interrupt
5477  */
ufshcd_uic_cmd_compl(struct ufs_hba * hba,u32 intr_status)5478 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
5479 {
5480 	irqreturn_t retval = IRQ_NONE;
5481 
5482 	spin_lock(hba->host->host_lock);
5483 	if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5484 		hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5485 
5486 	if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
5487 		hba->active_uic_cmd->argument2 |=
5488 			ufshcd_get_uic_cmd_result(hba);
5489 		hba->active_uic_cmd->argument3 =
5490 			ufshcd_get_dme_attr_val(hba);
5491 		if (!hba->uic_async_done)
5492 			hba->active_uic_cmd->cmd_active = 0;
5493 		complete(&hba->active_uic_cmd->done);
5494 		retval = IRQ_HANDLED;
5495 	}
5496 
5497 	if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
5498 		hba->active_uic_cmd->cmd_active = 0;
5499 		complete(hba->uic_async_done);
5500 		retval = IRQ_HANDLED;
5501 	}
5502 
5503 	if (retval == IRQ_HANDLED)
5504 		ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
5505 					     UFS_CMD_COMP);
5506 	spin_unlock(hba->host->host_lock);
5507 	return retval;
5508 }
5509 
5510 /* Release the resources allocated for processing a SCSI command. */
ufshcd_release_scsi_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)5511 void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
5512 			     struct ufshcd_lrb *lrbp)
5513 {
5514 	struct scsi_cmnd *cmd = lrbp->cmd;
5515 
5516 	scsi_dma_unmap(cmd);
5517 	ufshcd_crypto_clear_prdt(hba, lrbp);
5518 	ufshcd_release(hba);
5519 	ufshcd_clk_scaling_update_busy(hba);
5520 }
5521 
5522 /**
5523  * ufshcd_compl_one_cqe - handle a completion queue entry
5524  * @hba: per adapter instance
5525  * @task_tag: the task tag of the request to be completed
5526  * @cqe: pointer to the completion queue entry
5527  */
ufshcd_compl_one_cqe(struct ufs_hba * hba,int task_tag,struct cq_entry * cqe)5528 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
5529 			  struct cq_entry *cqe)
5530 {
5531 	struct ufshcd_lrb *lrbp;
5532 	struct scsi_cmnd *cmd;
5533 	enum utp_ocs ocs;
5534 
5535 	lrbp = &hba->lrb[task_tag];
5536 	lrbp->compl_time_stamp = ktime_get();
5537 	cmd = lrbp->cmd;
5538 	if (cmd) {
5539 		if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5540 			ufshcd_update_monitor(hba, lrbp);
5541 		ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP);
5542 		cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe);
5543 		ufshcd_release_scsi_cmd(hba, lrbp);
5544 		/* Do not touch lrbp after scsi done */
5545 		scsi_done(cmd);
5546 	} else if (hba->dev_cmd.complete) {
5547 		if (cqe) {
5548 			ocs = le32_to_cpu(cqe->status) & MASK_OCS;
5549 			lrbp->utr_descriptor_ptr->header.ocs = ocs;
5550 		}
5551 		complete(hba->dev_cmd.complete);
5552 	}
5553 }
5554 
5555 /**
5556  * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5557  * @hba: per adapter instance
5558  * @completed_reqs: bitmask that indicates which requests to complete
5559  */
__ufshcd_transfer_req_compl(struct ufs_hba * hba,unsigned long completed_reqs)5560 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5561 					unsigned long completed_reqs)
5562 {
5563 	int tag;
5564 
5565 	for_each_set_bit(tag, &completed_reqs, hba->nutrs)
5566 		ufshcd_compl_one_cqe(hba, tag, NULL);
5567 }
5568 
5569 /* Any value that is not an existing queue number is fine for this constant. */
5570 enum {
5571 	UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1
5572 };
5573 
ufshcd_clear_polled(struct ufs_hba * hba,unsigned long * completed_reqs)5574 static void ufshcd_clear_polled(struct ufs_hba *hba,
5575 				unsigned long *completed_reqs)
5576 {
5577 	int tag;
5578 
5579 	for_each_set_bit(tag, completed_reqs, hba->nutrs) {
5580 		struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
5581 
5582 		if (!cmd)
5583 			continue;
5584 		if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED)
5585 			__clear_bit(tag, completed_reqs);
5586 	}
5587 }
5588 
5589 /*
5590  * Return: > 0 if one or more commands have been completed or 0 if no
5591  * requests have been completed.
5592  */
ufshcd_poll(struct Scsi_Host * shost,unsigned int queue_num)5593 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
5594 {
5595 	struct ufs_hba *hba = shost_priv(shost);
5596 	unsigned long completed_reqs, flags;
5597 	u32 tr_doorbell;
5598 	struct ufs_hw_queue *hwq;
5599 
5600 	if (hba->mcq_enabled) {
5601 		hwq = &hba->uhq[queue_num];
5602 
5603 		return ufshcd_mcq_poll_cqe_lock(hba, hwq);
5604 	}
5605 
5606 	spin_lock_irqsave(&hba->outstanding_lock, flags);
5607 	tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5608 	completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
5609 	WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
5610 		  "completed: %#lx; outstanding: %#lx\n", completed_reqs,
5611 		  hba->outstanding_reqs);
5612 	if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) {
5613 		/* Do not complete polled requests from interrupt context. */
5614 		ufshcd_clear_polled(hba, &completed_reqs);
5615 	}
5616 	hba->outstanding_reqs &= ~completed_reqs;
5617 	spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5618 
5619 	if (completed_reqs)
5620 		__ufshcd_transfer_req_compl(hba, completed_reqs);
5621 
5622 	return completed_reqs != 0;
5623 }
5624 
5625 /**
5626  * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is
5627  * invoked from the error handler context or ufshcd_host_reset_and_restore()
5628  * to complete the pending transfers and free the resources associated with
5629  * the scsi command.
5630  *
5631  * @hba: per adapter instance
5632  * @force_compl: This flag is set to true when invoked
5633  * from ufshcd_host_reset_and_restore() in which case it requires special
5634  * handling because the host controller has been reset by ufshcd_hba_stop().
5635  */
ufshcd_mcq_compl_pending_transfer(struct ufs_hba * hba,bool force_compl)5636 static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba,
5637 					      bool force_compl)
5638 {
5639 	struct ufs_hw_queue *hwq;
5640 	struct ufshcd_lrb *lrbp;
5641 	struct scsi_cmnd *cmd;
5642 	unsigned long flags;
5643 	int tag;
5644 
5645 	for (tag = 0; tag < hba->nutrs; tag++) {
5646 		lrbp = &hba->lrb[tag];
5647 		cmd = lrbp->cmd;
5648 		if (!ufshcd_cmd_inflight(cmd) ||
5649 		    test_bit(SCMD_STATE_COMPLETE, &cmd->state))
5650 			continue;
5651 
5652 		hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
5653 
5654 		if (force_compl) {
5655 			ufshcd_mcq_compl_all_cqes_lock(hba, hwq);
5656 			/*
5657 			 * For those cmds of which the cqes are not present
5658 			 * in the cq, complete them explicitly.
5659 			 */
5660 			spin_lock_irqsave(&hwq->cq_lock, flags);
5661 			if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
5662 				set_host_byte(cmd, DID_REQUEUE);
5663 				ufshcd_release_scsi_cmd(hba, lrbp);
5664 				scsi_done(cmd);
5665 			}
5666 			spin_unlock_irqrestore(&hwq->cq_lock, flags);
5667 		} else {
5668 			ufshcd_mcq_poll_cqe_lock(hba, hwq);
5669 		}
5670 	}
5671 }
5672 
5673 /**
5674  * ufshcd_transfer_req_compl - handle SCSI and query command completion
5675  * @hba: per adapter instance
5676  *
5677  * Return:
5678  *  IRQ_HANDLED - If interrupt is valid
5679  *  IRQ_NONE    - If invalid interrupt
5680  */
ufshcd_transfer_req_compl(struct ufs_hba * hba)5681 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
5682 {
5683 	/* Resetting interrupt aggregation counters first and reading the
5684 	 * DOOR_BELL afterward allows us to handle all the completed requests.
5685 	 * In order to prevent other interrupts starvation the DB is read once
5686 	 * after reset. The down side of this solution is the possibility of
5687 	 * false interrupt if device completes another request after resetting
5688 	 * aggregation and before reading the DB.
5689 	 */
5690 	if (ufshcd_is_intr_aggr_allowed(hba) &&
5691 	    !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5692 		ufshcd_reset_intr_aggr(hba);
5693 
5694 	if (ufs_fail_completion(hba))
5695 		return IRQ_HANDLED;
5696 
5697 	/*
5698 	 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
5699 	 * do not want polling to trigger spurious interrupt complaints.
5700 	 */
5701 	ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
5702 
5703 	return IRQ_HANDLED;
5704 }
5705 
__ufshcd_write_ee_control(struct ufs_hba * hba,u32 ee_ctrl_mask)5706 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
5707 {
5708 	return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5709 				       QUERY_ATTR_IDN_EE_CONTROL, 0, 0,
5710 				       &ee_ctrl_mask);
5711 }
5712 
ufshcd_write_ee_control(struct ufs_hba * hba)5713 int ufshcd_write_ee_control(struct ufs_hba *hba)
5714 {
5715 	int err;
5716 
5717 	mutex_lock(&hba->ee_ctrl_mutex);
5718 	err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5719 	mutex_unlock(&hba->ee_ctrl_mutex);
5720 	if (err)
5721 		dev_err(hba->dev, "%s: failed to write ee control %d\n",
5722 			__func__, err);
5723 	return err;
5724 }
5725 
ufshcd_update_ee_control(struct ufs_hba * hba,u16 * mask,const u16 * other_mask,u16 set,u16 clr)5726 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask,
5727 			     const u16 *other_mask, u16 set, u16 clr)
5728 {
5729 	u16 new_mask, ee_ctrl_mask;
5730 	int err = 0;
5731 
5732 	mutex_lock(&hba->ee_ctrl_mutex);
5733 	new_mask = (*mask & ~clr) | set;
5734 	ee_ctrl_mask = new_mask | *other_mask;
5735 	if (ee_ctrl_mask != hba->ee_ctrl_mask)
5736 		err = __ufshcd_write_ee_control(hba, ee_ctrl_mask);
5737 	/* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */
5738 	if (!err) {
5739 		hba->ee_ctrl_mask = ee_ctrl_mask;
5740 		*mask = new_mask;
5741 	}
5742 	mutex_unlock(&hba->ee_ctrl_mutex);
5743 	return err;
5744 }
5745 
5746 /**
5747  * ufshcd_disable_ee - disable exception event
5748  * @hba: per-adapter instance
5749  * @mask: exception event to disable
5750  *
5751  * Disables exception event in the device so that the EVENT_ALERT
5752  * bit is not set.
5753  *
5754  * Return: zero on success, non-zero error value on failure.
5755  */
ufshcd_disable_ee(struct ufs_hba * hba,u16 mask)5756 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5757 {
5758 	return ufshcd_update_ee_drv_mask(hba, 0, mask);
5759 }
5760 
5761 /**
5762  * ufshcd_enable_ee - enable exception event
5763  * @hba: per-adapter instance
5764  * @mask: exception event to enable
5765  *
5766  * Enable corresponding exception event in the device to allow
5767  * device to alert host in critical scenarios.
5768  *
5769  * Return: zero on success, non-zero error value on failure.
5770  */
ufshcd_enable_ee(struct ufs_hba * hba,u16 mask)5771 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5772 {
5773 	return ufshcd_update_ee_drv_mask(hba, mask, 0);
5774 }
5775 
5776 /**
5777  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5778  * @hba: per-adapter instance
5779  *
5780  * Allow device to manage background operations on its own. Enabling
5781  * this might lead to inconsistent latencies during normal data transfers
5782  * as the device is allowed to manage its own way of handling background
5783  * operations.
5784  *
5785  * Return: zero on success, non-zero on failure.
5786  */
ufshcd_enable_auto_bkops(struct ufs_hba * hba)5787 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5788 {
5789 	int err = 0;
5790 
5791 	if (hba->auto_bkops_enabled)
5792 		goto out;
5793 
5794 	err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5795 			QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5796 	if (err) {
5797 		dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5798 				__func__, err);
5799 		goto out;
5800 	}
5801 
5802 	hba->auto_bkops_enabled = true;
5803 	trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5804 
5805 	/* No need of URGENT_BKOPS exception from the device */
5806 	err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5807 	if (err)
5808 		dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5809 				__func__, err);
5810 out:
5811 	return err;
5812 }
5813 
5814 /**
5815  * ufshcd_disable_auto_bkops - block device in doing background operations
5816  * @hba: per-adapter instance
5817  *
5818  * Disabling background operations improves command response latency but
5819  * has drawback of device moving into critical state where the device is
5820  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5821  * host is idle so that BKOPS are managed effectively without any negative
5822  * impacts.
5823  *
5824  * Return: zero on success, non-zero on failure.
5825  */
ufshcd_disable_auto_bkops(struct ufs_hba * hba)5826 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5827 {
5828 	int err = 0;
5829 
5830 	if (!hba->auto_bkops_enabled)
5831 		goto out;
5832 
5833 	/*
5834 	 * If host assisted BKOPs is to be enabled, make sure
5835 	 * urgent bkops exception is allowed.
5836 	 */
5837 	err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5838 	if (err) {
5839 		dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5840 				__func__, err);
5841 		goto out;
5842 	}
5843 
5844 	err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5845 			QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5846 	if (err) {
5847 		dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5848 				__func__, err);
5849 		ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5850 		goto out;
5851 	}
5852 
5853 	hba->auto_bkops_enabled = false;
5854 	trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5855 	hba->is_urgent_bkops_lvl_checked = false;
5856 out:
5857 	return err;
5858 }
5859 
5860 /**
5861  * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5862  * @hba: per adapter instance
5863  *
5864  * After a device reset the device may toggle the BKOPS_EN flag
5865  * to default value. The s/w tracking variables should be updated
5866  * as well. This function would change the auto-bkops state based on
5867  * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5868  */
ufshcd_force_reset_auto_bkops(struct ufs_hba * hba)5869 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5870 {
5871 	if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5872 		hba->auto_bkops_enabled = false;
5873 		hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5874 		ufshcd_enable_auto_bkops(hba);
5875 	} else {
5876 		hba->auto_bkops_enabled = true;
5877 		hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5878 		ufshcd_disable_auto_bkops(hba);
5879 	}
5880 	hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5881 	hba->is_urgent_bkops_lvl_checked = false;
5882 }
5883 
ufshcd_get_bkops_status(struct ufs_hba * hba,u32 * status)5884 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5885 {
5886 	return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5887 			QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5888 }
5889 
5890 /**
5891  * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5892  * @hba: per-adapter instance
5893  *
5894  * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5895  * flag in the device to permit background operations if the device
5896  * bkops_status is greater than or equal to the "hba->urgent_bkops_lvl",
5897  * disable otherwise.
5898  *
5899  * Return: 0 for success, non-zero in case of failure.
5900  *
5901  * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5902  * to know whether auto bkops is enabled or disabled after this function
5903  * returns control to it.
5904  */
ufshcd_bkops_ctrl(struct ufs_hba * hba)5905 static int ufshcd_bkops_ctrl(struct ufs_hba *hba)
5906 {
5907 	enum bkops_status status = hba->urgent_bkops_lvl;
5908 	u32 curr_status = 0;
5909 	int err;
5910 
5911 	err = ufshcd_get_bkops_status(hba, &curr_status);
5912 	if (err) {
5913 		dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5914 				__func__, err);
5915 		goto out;
5916 	} else if (curr_status > BKOPS_STATUS_MAX) {
5917 		dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5918 				__func__, curr_status);
5919 		err = -EINVAL;
5920 		goto out;
5921 	}
5922 
5923 	if (curr_status >= status)
5924 		err = ufshcd_enable_auto_bkops(hba);
5925 	else
5926 		err = ufshcd_disable_auto_bkops(hba);
5927 out:
5928 	return err;
5929 }
5930 
ufshcd_get_ee_status(struct ufs_hba * hba,u32 * status)5931 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5932 {
5933 	return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5934 			QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5935 }
5936 
ufshcd_bkops_exception_event_handler(struct ufs_hba * hba)5937 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5938 {
5939 	int err;
5940 	u32 curr_status = 0;
5941 
5942 	if (hba->is_urgent_bkops_lvl_checked)
5943 		goto enable_auto_bkops;
5944 
5945 	err = ufshcd_get_bkops_status(hba, &curr_status);
5946 	if (err) {
5947 		dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5948 				__func__, err);
5949 		goto out;
5950 	}
5951 
5952 	/*
5953 	 * We are seeing that some devices are raising the urgent bkops
5954 	 * exception events even when BKOPS status doesn't indicate performace
5955 	 * impacted or critical. Handle these device by determining their urgent
5956 	 * bkops status at runtime.
5957 	 */
5958 	if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5959 		dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5960 				__func__, curr_status);
5961 		/* update the current status as the urgent bkops level */
5962 		hba->urgent_bkops_lvl = curr_status;
5963 		hba->is_urgent_bkops_lvl_checked = true;
5964 	}
5965 
5966 enable_auto_bkops:
5967 	err = ufshcd_enable_auto_bkops(hba);
5968 out:
5969 	if (err < 0)
5970 		dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5971 				__func__, err);
5972 }
5973 
ufshcd_temp_exception_event_handler(struct ufs_hba * hba,u16 status)5974 static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status)
5975 {
5976 	u32 value;
5977 
5978 	if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5979 				QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value))
5980 		return;
5981 
5982 	dev_info(hba->dev, "exception Tcase %d\n", value - 80);
5983 
5984 	ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP);
5985 
5986 	/*
5987 	 * A placeholder for the platform vendors to add whatever additional
5988 	 * steps required
5989 	 */
5990 }
5991 
__ufshcd_wb_toggle(struct ufs_hba * hba,bool set,enum flag_idn idn)5992 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn)
5993 {
5994 	u8 index;
5995 	enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG :
5996 				   UPIU_QUERY_OPCODE_CLEAR_FLAG;
5997 
5998 	index = ufshcd_wb_get_query_index(hba);
5999 	return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL);
6000 }
6001 
ufshcd_wb_toggle(struct ufs_hba * hba,bool enable)6002 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable)
6003 {
6004 	int ret;
6005 
6006 	if (!ufshcd_is_wb_allowed(hba) ||
6007 	    hba->dev_info.wb_enabled == enable)
6008 		return 0;
6009 
6010 	ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN);
6011 	if (ret) {
6012 		dev_err(hba->dev, "%s: Write Booster %s failed %d\n",
6013 			__func__, enable ? "enabling" : "disabling", ret);
6014 		return ret;
6015 	}
6016 
6017 	hba->dev_info.wb_enabled = enable;
6018 	dev_dbg(hba->dev, "%s: Write Booster %s\n",
6019 			__func__, enable ? "enabled" : "disabled");
6020 
6021 	return ret;
6022 }
6023 
ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba * hba,bool enable)6024 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
6025 						 bool enable)
6026 {
6027 	int ret;
6028 
6029 	ret = __ufshcd_wb_toggle(hba, enable,
6030 			QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8);
6031 	if (ret) {
6032 		dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n",
6033 			__func__, enable ? "enabling" : "disabling", ret);
6034 		return;
6035 	}
6036 	dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n",
6037 			__func__, enable ? "enabled" : "disabled");
6038 }
6039 
ufshcd_wb_toggle_buf_flush(struct ufs_hba * hba,bool enable)6040 int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable)
6041 {
6042 	int ret;
6043 
6044 	if (!ufshcd_is_wb_allowed(hba) ||
6045 	    hba->dev_info.wb_buf_flush_enabled == enable)
6046 		return 0;
6047 
6048 	ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN);
6049 	if (ret) {
6050 		dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n",
6051 			__func__, enable ? "enabling" : "disabling", ret);
6052 		return ret;
6053 	}
6054 
6055 	hba->dev_info.wb_buf_flush_enabled = enable;
6056 	dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n",
6057 			__func__, enable ? "enabled" : "disabled");
6058 
6059 	return ret;
6060 }
6061 
ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba * hba,u32 avail_buf)6062 static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
6063 						u32 avail_buf)
6064 {
6065 	u32 cur_buf;
6066 	int ret;
6067 	u8 index;
6068 
6069 	index = ufshcd_wb_get_query_index(hba);
6070 	ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6071 					      QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
6072 					      index, 0, &cur_buf);
6073 	if (ret) {
6074 		dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n",
6075 			__func__, ret);
6076 		return false;
6077 	}
6078 
6079 	if (!cur_buf) {
6080 		dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
6081 			 cur_buf);
6082 		return false;
6083 	}
6084 	/* Let it continue to flush when available buffer exceeds threshold */
6085 	return avail_buf < hba->vps->wb_flush_threshold;
6086 }
6087 
ufshcd_wb_force_disable(struct ufs_hba * hba)6088 static void ufshcd_wb_force_disable(struct ufs_hba *hba)
6089 {
6090 	if (ufshcd_is_wb_buf_flush_allowed(hba))
6091 		ufshcd_wb_toggle_buf_flush(hba, false);
6092 
6093 	ufshcd_wb_toggle_buf_flush_during_h8(hba, false);
6094 	ufshcd_wb_toggle(hba, false);
6095 	hba->caps &= ~UFSHCD_CAP_WB_EN;
6096 
6097 	dev_info(hba->dev, "%s: WB force disabled\n", __func__);
6098 }
6099 
ufshcd_is_wb_buf_lifetime_available(struct ufs_hba * hba)6100 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba)
6101 {
6102 	u32 lifetime;
6103 	int ret;
6104 	u8 index;
6105 
6106 	index = ufshcd_wb_get_query_index(hba);
6107 	ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6108 				      QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST,
6109 				      index, 0, &lifetime);
6110 	if (ret) {
6111 		dev_err(hba->dev,
6112 			"%s: bWriteBoosterBufferLifeTimeEst read failed %d\n",
6113 			__func__, ret);
6114 		return false;
6115 	}
6116 
6117 	if (lifetime == UFS_WB_EXCEED_LIFETIME) {
6118 		dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n",
6119 			__func__, lifetime);
6120 		return false;
6121 	}
6122 
6123 	dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n",
6124 		__func__, lifetime);
6125 
6126 	return true;
6127 }
6128 
ufshcd_wb_need_flush(struct ufs_hba * hba)6129 static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
6130 {
6131 	int ret;
6132 	u32 avail_buf;
6133 	u8 index;
6134 
6135 	if (!ufshcd_is_wb_allowed(hba))
6136 		return false;
6137 
6138 	if (!ufshcd_is_wb_buf_lifetime_available(hba)) {
6139 		ufshcd_wb_force_disable(hba);
6140 		return false;
6141 	}
6142 
6143 	/*
6144 	 * The ufs device needs the vcc to be ON to flush.
6145 	 * With user-space reduction enabled, it's enough to enable flush
6146 	 * by checking only the available buffer. The threshold
6147 	 * defined here is > 90% full.
6148 	 * With user-space preserved enabled, the current-buffer
6149 	 * should be checked too because the wb buffer size can reduce
6150 	 * when disk tends to be full. This info is provided by current
6151 	 * buffer (dCurrentWriteBoosterBufferSize). There's no point in
6152 	 * keeping vcc on when current buffer is empty.
6153 	 */
6154 	index = ufshcd_wb_get_query_index(hba);
6155 	ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6156 				      QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
6157 				      index, 0, &avail_buf);
6158 	if (ret) {
6159 		dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n",
6160 			 __func__, ret);
6161 		return false;
6162 	}
6163 
6164 	if (!hba->dev_info.b_presrv_uspc_en)
6165 		return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10);
6166 
6167 	return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
6168 }
6169 
ufshcd_rpm_dev_flush_recheck_work(struct work_struct * work)6170 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
6171 {
6172 	struct ufs_hba *hba = container_of(to_delayed_work(work),
6173 					   struct ufs_hba,
6174 					   rpm_dev_flush_recheck_work);
6175 	/*
6176 	 * To prevent unnecessary VCC power drain after device finishes
6177 	 * WriteBooster buffer flush or Auto BKOPs, force runtime resume
6178 	 * after a certain delay to recheck the threshold by next runtime
6179 	 * suspend.
6180 	 */
6181 	ufshcd_rpm_get_sync(hba);
6182 	ufshcd_rpm_put_sync(hba);
6183 }
6184 
6185 /**
6186  * ufshcd_exception_event_handler - handle exceptions raised by device
6187  * @work: pointer to work data
6188  *
6189  * Read bExceptionEventStatus attribute from the device and handle the
6190  * exception event accordingly.
6191  */
ufshcd_exception_event_handler(struct work_struct * work)6192 static void ufshcd_exception_event_handler(struct work_struct *work)
6193 {
6194 	struct ufs_hba *hba;
6195 	int err;
6196 	u32 status = 0;
6197 	hba = container_of(work, struct ufs_hba, eeh_work);
6198 
6199 	ufshcd_scsi_block_requests(hba);
6200 	err = ufshcd_get_ee_status(hba, &status);
6201 	if (err) {
6202 		dev_err(hba->dev, "%s: failed to get exception status %d\n",
6203 				__func__, err);
6204 		goto out;
6205 	}
6206 
6207 	trace_ufshcd_exception_event(dev_name(hba->dev), status);
6208 
6209 	if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS)
6210 		ufshcd_bkops_exception_event_handler(hba);
6211 
6212 	if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP)
6213 		ufshcd_temp_exception_event_handler(hba, status);
6214 
6215 	ufs_debugfs_exception_event(hba, status);
6216 out:
6217 	ufshcd_scsi_unblock_requests(hba);
6218 }
6219 
6220 /* Complete requests that have door-bell cleared */
ufshcd_complete_requests(struct ufs_hba * hba,bool force_compl)6221 static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl)
6222 {
6223 	if (hba->mcq_enabled)
6224 		ufshcd_mcq_compl_pending_transfer(hba, force_compl);
6225 	else
6226 		ufshcd_transfer_req_compl(hba);
6227 
6228 	ufshcd_tmc_handler(hba);
6229 }
6230 
6231 /**
6232  * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
6233  *				to recover from the DL NAC errors or not.
6234  * @hba: per-adapter instance
6235  *
6236  * Return: true if error handling is required, false otherwise.
6237  */
ufshcd_quirk_dl_nac_errors(struct ufs_hba * hba)6238 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
6239 {
6240 	unsigned long flags;
6241 	bool err_handling = true;
6242 
6243 	spin_lock_irqsave(hba->host->host_lock, flags);
6244 	/*
6245 	 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
6246 	 * device fatal error and/or DL NAC & REPLAY timeout errors.
6247 	 */
6248 	if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
6249 		goto out;
6250 
6251 	if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
6252 	    ((hba->saved_err & UIC_ERROR) &&
6253 	     (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
6254 		goto out;
6255 
6256 	if ((hba->saved_err & UIC_ERROR) &&
6257 	    (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
6258 		int err;
6259 		/*
6260 		 * wait for 50ms to see if we can get any other errors or not.
6261 		 */
6262 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6263 		msleep(50);
6264 		spin_lock_irqsave(hba->host->host_lock, flags);
6265 
6266 		/*
6267 		 * now check if we have got any other severe errors other than
6268 		 * DL NAC error?
6269 		 */
6270 		if ((hba->saved_err & INT_FATAL_ERRORS) ||
6271 		    ((hba->saved_err & UIC_ERROR) &&
6272 		    (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
6273 			goto out;
6274 
6275 		/*
6276 		 * As DL NAC is the only error received so far, send out NOP
6277 		 * command to confirm if link is still active or not.
6278 		 *   - If we don't get any response then do error recovery.
6279 		 *   - If we get response then clear the DL NAC error bit.
6280 		 */
6281 
6282 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6283 		err = ufshcd_verify_dev_init(hba);
6284 		spin_lock_irqsave(hba->host->host_lock, flags);
6285 
6286 		if (err)
6287 			goto out;
6288 
6289 		/* Link seems to be alive hence ignore the DL NAC errors */
6290 		if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
6291 			hba->saved_err &= ~UIC_ERROR;
6292 		/* clear NAC error */
6293 		hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6294 		if (!hba->saved_uic_err)
6295 			err_handling = false;
6296 	}
6297 out:
6298 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6299 	return err_handling;
6300 }
6301 
6302 /* host lock must be held before calling this func */
ufshcd_is_saved_err_fatal(struct ufs_hba * hba)6303 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
6304 {
6305 	return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
6306 	       (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
6307 }
6308 
ufshcd_schedule_eh_work(struct ufs_hba * hba)6309 void ufshcd_schedule_eh_work(struct ufs_hba *hba)
6310 {
6311 	lockdep_assert_held(hba->host->host_lock);
6312 
6313 	/* handle fatal errors only when link is not in error state */
6314 	if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6315 		if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6316 		    ufshcd_is_saved_err_fatal(hba))
6317 			hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
6318 		else
6319 			hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
6320 		queue_work(hba->eh_wq, &hba->eh_work);
6321 	}
6322 }
6323 
ufshcd_force_error_recovery(struct ufs_hba * hba)6324 static void ufshcd_force_error_recovery(struct ufs_hba *hba)
6325 {
6326 	spin_lock_irq(hba->host->host_lock);
6327 	hba->force_reset = true;
6328 	ufshcd_schedule_eh_work(hba);
6329 	spin_unlock_irq(hba->host->host_lock);
6330 }
6331 
ufshcd_clk_scaling_allow(struct ufs_hba * hba,bool allow)6332 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
6333 {
6334 	mutex_lock(&hba->wb_mutex);
6335 	down_write(&hba->clk_scaling_lock);
6336 	hba->clk_scaling.is_allowed = allow;
6337 	up_write(&hba->clk_scaling_lock);
6338 	mutex_unlock(&hba->wb_mutex);
6339 }
6340 
ufshcd_clk_scaling_suspend(struct ufs_hba * hba,bool suspend)6341 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
6342 {
6343 	if (suspend) {
6344 		if (hba->clk_scaling.is_enabled)
6345 			ufshcd_suspend_clkscaling(hba);
6346 		ufshcd_clk_scaling_allow(hba, false);
6347 	} else {
6348 		ufshcd_clk_scaling_allow(hba, true);
6349 		if (hba->clk_scaling.is_enabled)
6350 			ufshcd_resume_clkscaling(hba);
6351 	}
6352 }
6353 
ufshcd_err_handling_prepare(struct ufs_hba * hba)6354 static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
6355 {
6356 	ufshcd_rpm_get_sync(hba);
6357 	if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) ||
6358 	    hba->is_sys_suspended) {
6359 		enum ufs_pm_op pm_op;
6360 
6361 		/*
6362 		 * Don't assume anything of resume, if
6363 		 * resume fails, irq and clocks can be OFF, and powers
6364 		 * can be OFF or in LPM.
6365 		 */
6366 		ufshcd_setup_hba_vreg(hba, true);
6367 		ufshcd_enable_irq(hba);
6368 		ufshcd_setup_vreg(hba, true);
6369 		ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
6370 		ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
6371 		ufshcd_hold(hba);
6372 		if (!ufshcd_is_clkgating_allowed(hba))
6373 			ufshcd_setup_clocks(hba, true);
6374 		pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
6375 		ufshcd_vops_resume(hba, pm_op);
6376 	} else {
6377 		ufshcd_hold(hba);
6378 		if (ufshcd_is_clkscaling_supported(hba) &&
6379 		    hba->clk_scaling.is_enabled)
6380 			ufshcd_suspend_clkscaling(hba);
6381 		ufshcd_clk_scaling_allow(hba, false);
6382 	}
6383 	ufshcd_scsi_block_requests(hba);
6384 	/* Wait for ongoing ufshcd_queuecommand() calls to finish. */
6385 	blk_mq_wait_quiesce_done(&hba->host->tag_set);
6386 	cancel_work_sync(&hba->eeh_work);
6387 }
6388 
ufshcd_err_handling_unprepare(struct ufs_hba * hba)6389 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
6390 {
6391 	ufshcd_scsi_unblock_requests(hba);
6392 	ufshcd_release(hba);
6393 	if (ufshcd_is_clkscaling_supported(hba))
6394 		ufshcd_clk_scaling_suspend(hba, false);
6395 	ufshcd_rpm_put(hba);
6396 }
6397 
ufshcd_err_handling_should_stop(struct ufs_hba * hba)6398 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
6399 {
6400 	return (!hba->is_powered || hba->shutting_down ||
6401 		!hba->ufs_device_wlun ||
6402 		hba->ufshcd_state == UFSHCD_STATE_ERROR ||
6403 		(!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
6404 		   ufshcd_is_link_broken(hba))));
6405 }
6406 
6407 #ifdef CONFIG_PM
ufshcd_recover_pm_error(struct ufs_hba * hba)6408 static void ufshcd_recover_pm_error(struct ufs_hba *hba)
6409 {
6410 	struct Scsi_Host *shost = hba->host;
6411 	struct scsi_device *sdev;
6412 	struct request_queue *q;
6413 	int ret;
6414 
6415 	hba->is_sys_suspended = false;
6416 	/*
6417 	 * Set RPM status of wlun device to RPM_ACTIVE,
6418 	 * this also clears its runtime error.
6419 	 */
6420 	ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev);
6421 
6422 	/* hba device might have a runtime error otherwise */
6423 	if (ret)
6424 		ret = pm_runtime_set_active(hba->dev);
6425 	/*
6426 	 * If wlun device had runtime error, we also need to resume those
6427 	 * consumer scsi devices in case any of them has failed to be
6428 	 * resumed due to supplier runtime resume failure. This is to unblock
6429 	 * blk_queue_enter in case there are bios waiting inside it.
6430 	 */
6431 	if (!ret) {
6432 		shost_for_each_device(sdev, shost) {
6433 			q = sdev->request_queue;
6434 			if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
6435 				       q->rpm_status == RPM_SUSPENDING))
6436 				pm_request_resume(q->dev);
6437 		}
6438 	}
6439 }
6440 #else
ufshcd_recover_pm_error(struct ufs_hba * hba)6441 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
6442 {
6443 }
6444 #endif
6445 
ufshcd_is_pwr_mode_restore_needed(struct ufs_hba * hba)6446 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
6447 {
6448 	struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
6449 	u32 mode;
6450 
6451 	ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
6452 
6453 	if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
6454 		return true;
6455 
6456 	if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
6457 		return true;
6458 
6459 	return false;
6460 }
6461 
ufshcd_abort_one(struct request * rq,void * priv)6462 static bool ufshcd_abort_one(struct request *rq, void *priv)
6463 {
6464 	int *ret = priv;
6465 	u32 tag = rq->tag;
6466 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
6467 	struct scsi_device *sdev = cmd->device;
6468 	struct Scsi_Host *shost = sdev->host;
6469 	struct ufs_hba *hba = shost_priv(shost);
6470 
6471 	*ret = ufshcd_try_to_abort_task(hba, tag);
6472 	dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
6473 		hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
6474 		*ret ? "failed" : "succeeded");
6475 
6476 	return *ret == 0;
6477 }
6478 
6479 /**
6480  * ufshcd_abort_all - Abort all pending commands.
6481  * @hba: Host bus adapter pointer.
6482  *
6483  * Return: true if and only if the host controller needs to be reset.
6484  */
ufshcd_abort_all(struct ufs_hba * hba)6485 static bool ufshcd_abort_all(struct ufs_hba *hba)
6486 {
6487 	int tag, ret = 0;
6488 
6489 	blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret);
6490 	if (ret)
6491 		goto out;
6492 
6493 	/* Clear pending task management requests */
6494 	for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6495 		ret = ufshcd_clear_tm_cmd(hba, tag);
6496 		if (ret)
6497 			goto out;
6498 	}
6499 
6500 out:
6501 	/* Complete the requests that are cleared by s/w */
6502 	ufshcd_complete_requests(hba, false);
6503 
6504 	return ret != 0;
6505 }
6506 
6507 /**
6508  * ufshcd_err_handler - handle UFS errors that require s/w attention
6509  * @work: pointer to work structure
6510  */
ufshcd_err_handler(struct work_struct * work)6511 static void ufshcd_err_handler(struct work_struct *work)
6512 {
6513 	int retries = MAX_ERR_HANDLER_RETRIES;
6514 	struct ufs_hba *hba;
6515 	unsigned long flags;
6516 	bool needs_restore;
6517 	bool needs_reset;
6518 	int pmc_err;
6519 
6520 	hba = container_of(work, struct ufs_hba, eh_work);
6521 
6522 	dev_info(hba->dev,
6523 		 "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n",
6524 		 __func__, ufshcd_state_name[hba->ufshcd_state],
6525 		 hba->is_powered, hba->shutting_down, hba->saved_err,
6526 		 hba->saved_uic_err, hba->force_reset,
6527 		 ufshcd_is_link_broken(hba) ? "; link is broken" : "");
6528 
6529 	down(&hba->host_sem);
6530 	spin_lock_irqsave(hba->host->host_lock, flags);
6531 	if (ufshcd_err_handling_should_stop(hba)) {
6532 		if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6533 			hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6534 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6535 		up(&hba->host_sem);
6536 		return;
6537 	}
6538 	ufshcd_set_eh_in_progress(hba);
6539 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6540 	ufshcd_err_handling_prepare(hba);
6541 	/* Complete requests that have door-bell cleared by h/w */
6542 	ufshcd_complete_requests(hba, false);
6543 	spin_lock_irqsave(hba->host->host_lock, flags);
6544 again:
6545 	needs_restore = false;
6546 	needs_reset = false;
6547 
6548 	if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6549 		hba->ufshcd_state = UFSHCD_STATE_RESET;
6550 	/*
6551 	 * A full reset and restore might have happened after preparation
6552 	 * is finished, double check whether we should stop.
6553 	 */
6554 	if (ufshcd_err_handling_should_stop(hba))
6555 		goto skip_err_handling;
6556 
6557 	if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) &&
6558 	    !hba->force_reset) {
6559 		bool ret;
6560 
6561 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6562 		/* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
6563 		ret = ufshcd_quirk_dl_nac_errors(hba);
6564 		spin_lock_irqsave(hba->host->host_lock, flags);
6565 		if (!ret && ufshcd_err_handling_should_stop(hba))
6566 			goto skip_err_handling;
6567 	}
6568 
6569 	if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6570 	    (hba->saved_uic_err &&
6571 	     (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6572 		bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6573 
6574 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6575 		ufshcd_print_host_state(hba);
6576 		ufshcd_print_pwr_info(hba);
6577 		ufshcd_print_evt_hist(hba);
6578 		ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6579 		ufshcd_print_trs_all(hba, pr_prdt);
6580 		spin_lock_irqsave(hba->host->host_lock, flags);
6581 	}
6582 
6583 	/*
6584 	 * if host reset is required then skip clearing the pending
6585 	 * transfers forcefully because they will get cleared during
6586 	 * host reset and restore
6587 	 */
6588 	if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6589 	    ufshcd_is_saved_err_fatal(hba) ||
6590 	    ((hba->saved_err & UIC_ERROR) &&
6591 	     (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6592 				    UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
6593 		needs_reset = true;
6594 		goto do_reset;
6595 	}
6596 
6597 	/*
6598 	 * If LINERESET was caught, UFS might have been put to PWM mode,
6599 	 * check if power mode restore is needed.
6600 	 */
6601 	if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6602 		hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6603 		if (!hba->saved_uic_err)
6604 			hba->saved_err &= ~UIC_ERROR;
6605 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6606 		if (ufshcd_is_pwr_mode_restore_needed(hba))
6607 			needs_restore = true;
6608 		spin_lock_irqsave(hba->host->host_lock, flags);
6609 		if (!hba->saved_err && !needs_restore)
6610 			goto skip_err_handling;
6611 	}
6612 
6613 	hba->silence_err_logs = true;
6614 	/* release lock as clear command might sleep */
6615 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6616 
6617 	needs_reset = ufshcd_abort_all(hba);
6618 
6619 	spin_lock_irqsave(hba->host->host_lock, flags);
6620 	hba->silence_err_logs = false;
6621 	if (needs_reset)
6622 		goto do_reset;
6623 
6624 	/*
6625 	 * After all reqs and tasks are cleared from doorbell,
6626 	 * now it is safe to retore power mode.
6627 	 */
6628 	if (needs_restore) {
6629 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6630 		/*
6631 		 * Hold the scaling lock just in case dev cmds
6632 		 * are sent via bsg and/or sysfs.
6633 		 */
6634 		down_write(&hba->clk_scaling_lock);
6635 		hba->force_pmc = true;
6636 		pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6637 		if (pmc_err) {
6638 			needs_reset = true;
6639 			dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6640 					__func__, pmc_err);
6641 		}
6642 		hba->force_pmc = false;
6643 		ufshcd_print_pwr_info(hba);
6644 		up_write(&hba->clk_scaling_lock);
6645 		spin_lock_irqsave(hba->host->host_lock, flags);
6646 	}
6647 
6648 do_reset:
6649 	/* Fatal errors need reset */
6650 	if (needs_reset) {
6651 		int err;
6652 
6653 		hba->force_reset = false;
6654 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6655 		err = ufshcd_reset_and_restore(hba);
6656 		if (err)
6657 			dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6658 					__func__, err);
6659 		else
6660 			ufshcd_recover_pm_error(hba);
6661 		spin_lock_irqsave(hba->host->host_lock, flags);
6662 	}
6663 
6664 skip_err_handling:
6665 	if (!needs_reset) {
6666 		if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6667 			hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6668 		if (hba->saved_err || hba->saved_uic_err)
6669 			dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6670 			    __func__, hba->saved_err, hba->saved_uic_err);
6671 	}
6672 	/* Exit in an operational state or dead */
6673 	if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
6674 	    hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6675 		if (--retries)
6676 			goto again;
6677 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
6678 	}
6679 	ufshcd_clear_eh_in_progress(hba);
6680 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6681 	ufshcd_err_handling_unprepare(hba);
6682 	up(&hba->host_sem);
6683 
6684 	dev_info(hba->dev, "%s finished; HBA state %s\n", __func__,
6685 		 ufshcd_state_name[hba->ufshcd_state]);
6686 }
6687 
6688 /**
6689  * ufshcd_update_uic_error - check and set fatal UIC error flags.
6690  * @hba: per-adapter instance
6691  *
6692  * Return:
6693  *  IRQ_HANDLED - If interrupt is valid
6694  *  IRQ_NONE    - If invalid interrupt
6695  */
ufshcd_update_uic_error(struct ufs_hba * hba)6696 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
6697 {
6698 	u32 reg;
6699 	irqreturn_t retval = IRQ_NONE;
6700 
6701 	/* PHY layer error */
6702 	reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
6703 	if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
6704 	    (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
6705 		ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
6706 		/*
6707 		 * To know whether this error is fatal or not, DB timeout
6708 		 * must be checked but this error is handled separately.
6709 		 */
6710 		if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6711 			dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6712 					__func__);
6713 
6714 		/* Got a LINERESET indication. */
6715 		if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6716 			struct uic_command *cmd = NULL;
6717 
6718 			hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6719 			if (hba->uic_async_done && hba->active_uic_cmd)
6720 				cmd = hba->active_uic_cmd;
6721 			/*
6722 			 * Ignore the LINERESET during power mode change
6723 			 * operation via DME_SET command.
6724 			 */
6725 			if (cmd && (cmd->command == UIC_CMD_DME_SET))
6726 				hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6727 		}
6728 		retval |= IRQ_HANDLED;
6729 	}
6730 
6731 	/* PA_INIT_ERROR is fatal and needs UIC reset */
6732 	reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
6733 	if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6734 	    (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
6735 		ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
6736 
6737 		if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6738 			hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6739 		else if (hba->dev_quirks &
6740 				UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6741 			if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6742 				hba->uic_error |=
6743 					UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6744 			else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6745 				hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6746 		}
6747 		retval |= IRQ_HANDLED;
6748 	}
6749 
6750 	/* UIC NL/TL/DME errors needs software retry */
6751 	reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
6752 	if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6753 	    (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
6754 		ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
6755 		hba->uic_error |= UFSHCD_UIC_NL_ERROR;
6756 		retval |= IRQ_HANDLED;
6757 	}
6758 
6759 	reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
6760 	if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6761 	    (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
6762 		ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
6763 		hba->uic_error |= UFSHCD_UIC_TL_ERROR;
6764 		retval |= IRQ_HANDLED;
6765 	}
6766 
6767 	reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
6768 	if ((reg & UIC_DME_ERROR) &&
6769 	    (reg & UIC_DME_ERROR_CODE_MASK)) {
6770 		ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
6771 		hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6772 		retval |= IRQ_HANDLED;
6773 	}
6774 
6775 	dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6776 			__func__, hba->uic_error);
6777 	return retval;
6778 }
6779 
6780 /**
6781  * ufshcd_check_errors - Check for errors that need s/w attention
6782  * @hba: per-adapter instance
6783  * @intr_status: interrupt status generated by the controller
6784  *
6785  * Return:
6786  *  IRQ_HANDLED - If interrupt is valid
6787  *  IRQ_NONE    - If invalid interrupt
6788  */
ufshcd_check_errors(struct ufs_hba * hba,u32 intr_status)6789 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
6790 {
6791 	bool queue_eh_work = false;
6792 	irqreturn_t retval = IRQ_NONE;
6793 
6794 	spin_lock(hba->host->host_lock);
6795 	hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6796 
6797 	if (hba->errors & INT_FATAL_ERRORS) {
6798 		ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6799 				       hba->errors);
6800 		queue_eh_work = true;
6801 	}
6802 
6803 	if (hba->errors & UIC_ERROR) {
6804 		hba->uic_error = 0;
6805 		retval = ufshcd_update_uic_error(hba);
6806 		if (hba->uic_error)
6807 			queue_eh_work = true;
6808 	}
6809 
6810 	if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6811 		dev_err(hba->dev,
6812 			"%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6813 			__func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6814 			"Enter" : "Exit",
6815 			hba->errors, ufshcd_get_upmcrs(hba));
6816 		ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
6817 				       hba->errors);
6818 		ufshcd_set_link_broken(hba);
6819 		queue_eh_work = true;
6820 	}
6821 
6822 	if (queue_eh_work) {
6823 		/*
6824 		 * update the transfer error masks to sticky bits, let's do this
6825 		 * irrespective of current ufshcd_state.
6826 		 */
6827 		hba->saved_err |= hba->errors;
6828 		hba->saved_uic_err |= hba->uic_error;
6829 
6830 		/* dump controller state before resetting */
6831 		if ((hba->saved_err &
6832 		     (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6833 		    (hba->saved_uic_err &&
6834 		     (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6835 			dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6836 					__func__, hba->saved_err,
6837 					hba->saved_uic_err);
6838 			ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6839 					 "host_regs: ");
6840 			ufshcd_print_pwr_info(hba);
6841 		}
6842 		ufshcd_schedule_eh_work(hba);
6843 		retval |= IRQ_HANDLED;
6844 	}
6845 	/*
6846 	 * if (!queue_eh_work) -
6847 	 * Other errors are either non-fatal where host recovers
6848 	 * itself without s/w intervention or errors that will be
6849 	 * handled by the SCSI core layer.
6850 	 */
6851 	hba->errors = 0;
6852 	hba->uic_error = 0;
6853 	spin_unlock(hba->host->host_lock);
6854 	return retval;
6855 }
6856 
6857 /**
6858  * ufshcd_tmc_handler - handle task management function completion
6859  * @hba: per adapter instance
6860  *
6861  * Return:
6862  *  IRQ_HANDLED - If interrupt is valid
6863  *  IRQ_NONE    - If invalid interrupt
6864  */
ufshcd_tmc_handler(struct ufs_hba * hba)6865 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
6866 {
6867 	unsigned long flags, pending, issued;
6868 	irqreturn_t ret = IRQ_NONE;
6869 	int tag;
6870 
6871 	spin_lock_irqsave(hba->host->host_lock, flags);
6872 	pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
6873 	issued = hba->outstanding_tasks & ~pending;
6874 	for_each_set_bit(tag, &issued, hba->nutmrs) {
6875 		struct request *req = hba->tmf_rqs[tag];
6876 		struct completion *c = req->end_io_data;
6877 
6878 		complete(c);
6879 		ret = IRQ_HANDLED;
6880 	}
6881 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6882 
6883 	return ret;
6884 }
6885 
6886 /**
6887  * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events
6888  * @hba: per adapter instance
6889  *
6890  * Return: IRQ_HANDLED if interrupt is handled.
6891  */
ufshcd_handle_mcq_cq_events(struct ufs_hba * hba)6892 static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba)
6893 {
6894 	struct ufs_hw_queue *hwq;
6895 	unsigned long outstanding_cqs;
6896 	unsigned int nr_queues;
6897 	int i, ret;
6898 	u32 events;
6899 
6900 	ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs);
6901 	if (ret)
6902 		outstanding_cqs = (1U << hba->nr_hw_queues) - 1;
6903 
6904 	/* Exclude the poll queues */
6905 	nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL];
6906 	for_each_set_bit(i, &outstanding_cqs, nr_queues) {
6907 		hwq = &hba->uhq[i];
6908 
6909 		events = ufshcd_mcq_read_cqis(hba, i);
6910 		if (events)
6911 			ufshcd_mcq_write_cqis(hba, events, i);
6912 
6913 		if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
6914 			ufshcd_mcq_poll_cqe_lock(hba, hwq);
6915 	}
6916 
6917 	return IRQ_HANDLED;
6918 }
6919 
6920 /**
6921  * ufshcd_sl_intr - Interrupt service routine
6922  * @hba: per adapter instance
6923  * @intr_status: contains interrupts generated by the controller
6924  *
6925  * Return:
6926  *  IRQ_HANDLED - If interrupt is valid
6927  *  IRQ_NONE    - If invalid interrupt
6928  */
ufshcd_sl_intr(struct ufs_hba * hba,u32 intr_status)6929 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
6930 {
6931 	irqreturn_t retval = IRQ_NONE;
6932 
6933 	if (intr_status & UFSHCD_UIC_MASK)
6934 		retval |= ufshcd_uic_cmd_compl(hba, intr_status);
6935 
6936 	if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6937 		retval |= ufshcd_check_errors(hba, intr_status);
6938 
6939 	if (intr_status & UTP_TASK_REQ_COMPL)
6940 		retval |= ufshcd_tmc_handler(hba);
6941 
6942 	if (intr_status & UTP_TRANSFER_REQ_COMPL)
6943 		retval |= ufshcd_transfer_req_compl(hba);
6944 
6945 	if (intr_status & MCQ_CQ_EVENT_STATUS)
6946 		retval |= ufshcd_handle_mcq_cq_events(hba);
6947 
6948 	return retval;
6949 }
6950 
6951 /**
6952  * ufshcd_intr - Main interrupt service routine
6953  * @irq: irq number
6954  * @__hba: pointer to adapter instance
6955  *
6956  * Return:
6957  *  IRQ_HANDLED - If interrupt is valid
6958  *  IRQ_NONE    - If invalid interrupt
6959  */
ufshcd_intr(int irq,void * __hba)6960 static irqreturn_t ufshcd_intr(int irq, void *__hba)
6961 {
6962 	u32 intr_status, enabled_intr_status = 0;
6963 	irqreturn_t retval = IRQ_NONE;
6964 	struct ufs_hba *hba = __hba;
6965 	int retries = hba->nutrs;
6966 
6967 	intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6968 	hba->ufs_stats.last_intr_status = intr_status;
6969 	hba->ufs_stats.last_intr_ts = local_clock();
6970 
6971 	/*
6972 	 * There could be max of hba->nutrs reqs in flight and in worst case
6973 	 * if the reqs get finished 1 by 1 after the interrupt status is
6974 	 * read, make sure we handle them by checking the interrupt status
6975 	 * again in a loop until we process all of the reqs before returning.
6976 	 */
6977 	while (intr_status && retries--) {
6978 		enabled_intr_status =
6979 			intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6980 		ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
6981 		if (enabled_intr_status)
6982 			retval |= ufshcd_sl_intr(hba, enabled_intr_status);
6983 
6984 		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6985 	}
6986 
6987 	if (enabled_intr_status && retval == IRQ_NONE &&
6988 	    (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) ||
6989 	     hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) {
6990 		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6991 					__func__,
6992 					intr_status,
6993 					hba->ufs_stats.last_intr_status,
6994 					enabled_intr_status);
6995 		ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6996 	}
6997 
6998 	return retval;
6999 }
7000 
ufshcd_clear_tm_cmd(struct ufs_hba * hba,int tag)7001 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
7002 {
7003 	int err = 0;
7004 	u32 mask = 1 << tag;
7005 	unsigned long flags;
7006 
7007 	if (!test_bit(tag, &hba->outstanding_tasks))
7008 		goto out;
7009 
7010 	spin_lock_irqsave(hba->host->host_lock, flags);
7011 	ufshcd_utmrl_clear(hba, tag);
7012 	spin_unlock_irqrestore(hba->host->host_lock, flags);
7013 
7014 	/* poll for max. 1 sec to clear door bell register by h/w */
7015 	err = ufshcd_wait_for_register(hba,
7016 			REG_UTP_TASK_REQ_DOOR_BELL,
7017 			mask, 0, 1000, 1000);
7018 
7019 	dev_err(hba->dev, "Clearing task management function with tag %d %s\n",
7020 		tag, err < 0 ? "failed" : "succeeded");
7021 
7022 out:
7023 	return err;
7024 }
7025 
__ufshcd_issue_tm_cmd(struct ufs_hba * hba,struct utp_task_req_desc * treq,u8 tm_function)7026 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
7027 		struct utp_task_req_desc *treq, u8 tm_function)
7028 {
7029 	struct request_queue *q = hba->tmf_queue;
7030 	struct Scsi_Host *host = hba->host;
7031 	DECLARE_COMPLETION_ONSTACK(wait);
7032 	struct request *req;
7033 	unsigned long flags;
7034 	int task_tag, err;
7035 
7036 	/*
7037 	 * blk_mq_alloc_request() is used here only to get a free tag.
7038 	 */
7039 	req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0);
7040 	if (IS_ERR(req))
7041 		return PTR_ERR(req);
7042 
7043 	req->end_io_data = &wait;
7044 	ufshcd_hold(hba);
7045 
7046 	spin_lock_irqsave(host->host_lock, flags);
7047 
7048 	task_tag = req->tag;
7049 	hba->tmf_rqs[req->tag] = req;
7050 	treq->upiu_req.req_header.task_tag = task_tag;
7051 
7052 	memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
7053 	ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function);
7054 
7055 	/* send command to the controller */
7056 	__set_bit(task_tag, &hba->outstanding_tasks);
7057 	ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
7058 
7059 	spin_unlock_irqrestore(host->host_lock, flags);
7060 
7061 	ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND);
7062 
7063 	/* wait until the task management command is completed */
7064 	err = wait_for_completion_io_timeout(&wait,
7065 			msecs_to_jiffies(TM_CMD_TIMEOUT));
7066 	if (!err) {
7067 		ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
7068 		dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
7069 				__func__, tm_function);
7070 		if (ufshcd_clear_tm_cmd(hba, task_tag))
7071 			dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
7072 					__func__, task_tag);
7073 		err = -ETIMEDOUT;
7074 	} else {
7075 		err = 0;
7076 		memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
7077 
7078 		ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
7079 	}
7080 
7081 	spin_lock_irqsave(hba->host->host_lock, flags);
7082 	hba->tmf_rqs[req->tag] = NULL;
7083 	__clear_bit(task_tag, &hba->outstanding_tasks);
7084 	spin_unlock_irqrestore(hba->host->host_lock, flags);
7085 
7086 	ufshcd_release(hba);
7087 	blk_mq_free_request(req);
7088 
7089 	return err;
7090 }
7091 
7092 /**
7093  * ufshcd_issue_tm_cmd - issues task management commands to controller
7094  * @hba: per adapter instance
7095  * @lun_id: LUN ID to which TM command is sent
7096  * @task_id: task ID to which the TM command is applicable
7097  * @tm_function: task management function opcode
7098  * @tm_response: task management service response return value
7099  *
7100  * Return: non-zero value on error, zero on success.
7101  */
ufshcd_issue_tm_cmd(struct ufs_hba * hba,int lun_id,int task_id,u8 tm_function,u8 * tm_response)7102 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
7103 		u8 tm_function, u8 *tm_response)
7104 {
7105 	struct utp_task_req_desc treq = { };
7106 	enum utp_ocs ocs_value;
7107 	int err;
7108 
7109 	/* Configure task request descriptor */
7110 	treq.header.interrupt = 1;
7111 	treq.header.ocs = OCS_INVALID_COMMAND_STATUS;
7112 
7113 	/* Configure task request UPIU */
7114 	treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ;
7115 	treq.upiu_req.req_header.lun = lun_id;
7116 	treq.upiu_req.req_header.tm_function = tm_function;
7117 
7118 	/*
7119 	 * The host shall provide the same value for LUN field in the basic
7120 	 * header and for Input Parameter.
7121 	 */
7122 	treq.upiu_req.input_param1 = cpu_to_be32(lun_id);
7123 	treq.upiu_req.input_param2 = cpu_to_be32(task_id);
7124 
7125 	err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
7126 	if (err == -ETIMEDOUT)
7127 		return err;
7128 
7129 	ocs_value = treq.header.ocs & MASK_OCS;
7130 	if (ocs_value != OCS_SUCCESS)
7131 		dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
7132 				__func__, ocs_value);
7133 	else if (tm_response)
7134 		*tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) &
7135 				MASK_TM_SERVICE_RESP;
7136 	return err;
7137 }
7138 
7139 /**
7140  * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
7141  * @hba:	per-adapter instance
7142  * @req_upiu:	upiu request
7143  * @rsp_upiu:	upiu reply
7144  * @desc_buff:	pointer to descriptor buffer, NULL if NA
7145  * @buff_len:	descriptor size, 0 if NA
7146  * @cmd_type:	specifies the type (NOP, Query...)
7147  * @desc_op:	descriptor operation
7148  *
7149  * Those type of requests uses UTP Transfer Request Descriptor - utrd.
7150  * Therefore, it "rides" the device management infrastructure: uses its tag and
7151  * tasks work queues.
7152  *
7153  * Since there is only one available tag for device management commands,
7154  * the caller is expected to hold the hba->dev_cmd.lock mutex.
7155  *
7156  * Return: 0 upon success; < 0 upon failure.
7157  */
ufshcd_issue_devman_upiu_cmd(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,u8 * desc_buff,int * buff_len,enum dev_cmd_type cmd_type,enum query_opcode desc_op)7158 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
7159 					struct utp_upiu_req *req_upiu,
7160 					struct utp_upiu_req *rsp_upiu,
7161 					u8 *desc_buff, int *buff_len,
7162 					enum dev_cmd_type cmd_type,
7163 					enum query_opcode desc_op)
7164 {
7165 	const u32 tag = hba->reserved_slot;
7166 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7167 	int err = 0;
7168 	u8 upiu_flags;
7169 
7170 	/* Protects use of hba->reserved_slot. */
7171 	lockdep_assert_held(&hba->dev_cmd.lock);
7172 
7173 	ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
7174 
7175 	ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
7176 
7177 	/* update the task tag in the request upiu */
7178 	req_upiu->header.task_tag = tag;
7179 
7180 	/* just copy the upiu request as it is */
7181 	memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7182 	if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
7183 		/* The Data Segment Area is optional depending upon the query
7184 		 * function value. for WRITE DESCRIPTOR, the data segment
7185 		 * follows right after the tsf.
7186 		 */
7187 		memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
7188 		*buff_len = 0;
7189 	}
7190 
7191 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7192 
7193 	/*
7194 	 * ignore the returning value here - ufshcd_check_query_response is
7195 	 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
7196 	 * read the response directly ignoring all errors.
7197 	 */
7198 	ufshcd_issue_dev_cmd(hba, lrbp, tag, QUERY_REQ_TIMEOUT);
7199 
7200 	/* just copy the upiu response as it is */
7201 	memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7202 	if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
7203 		u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
7204 		u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
7205 					   .data_segment_length);
7206 
7207 		if (*buff_len >= resp_len) {
7208 			memcpy(desc_buff, descp, resp_len);
7209 			*buff_len = resp_len;
7210 		} else {
7211 			dev_warn(hba->dev,
7212 				 "%s: rsp size %d is bigger than buffer size %d",
7213 				 __func__, resp_len, *buff_len);
7214 			*buff_len = 0;
7215 			err = -EINVAL;
7216 		}
7217 	}
7218 	ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
7219 				    (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
7220 
7221 	return err;
7222 }
7223 
7224 /**
7225  * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
7226  * @hba:	per-adapter instance
7227  * @req_upiu:	upiu request
7228  * @rsp_upiu:	upiu reply - only 8 DW as we do not support scsi commands
7229  * @msgcode:	message code, one of UPIU Transaction Codes Initiator to Target
7230  * @desc_buff:	pointer to descriptor buffer, NULL if NA
7231  * @buff_len:	descriptor size, 0 if NA
7232  * @desc_op:	descriptor operation
7233  *
7234  * Supports UTP Transfer requests (nop and query), and UTP Task
7235  * Management requests.
7236  * It is up to the caller to fill the upiu conent properly, as it will
7237  * be copied without any further input validations.
7238  *
7239  * Return: 0 upon success; < 0 upon failure.
7240  */
ufshcd_exec_raw_upiu_cmd(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,enum upiu_request_transaction msgcode,u8 * desc_buff,int * buff_len,enum query_opcode desc_op)7241 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
7242 			     struct utp_upiu_req *req_upiu,
7243 			     struct utp_upiu_req *rsp_upiu,
7244 			     enum upiu_request_transaction msgcode,
7245 			     u8 *desc_buff, int *buff_len,
7246 			     enum query_opcode desc_op)
7247 {
7248 	int err;
7249 	enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
7250 	struct utp_task_req_desc treq = { };
7251 	enum utp_ocs ocs_value;
7252 	u8 tm_f = req_upiu->header.tm_function;
7253 
7254 	switch (msgcode) {
7255 	case UPIU_TRANSACTION_NOP_OUT:
7256 		cmd_type = DEV_CMD_TYPE_NOP;
7257 		fallthrough;
7258 	case UPIU_TRANSACTION_QUERY_REQ:
7259 		ufshcd_dev_man_lock(hba);
7260 		err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
7261 						   desc_buff, buff_len,
7262 						   cmd_type, desc_op);
7263 		ufshcd_dev_man_unlock(hba);
7264 
7265 		break;
7266 	case UPIU_TRANSACTION_TASK_REQ:
7267 		treq.header.interrupt = 1;
7268 		treq.header.ocs = OCS_INVALID_COMMAND_STATUS;
7269 
7270 		memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu));
7271 
7272 		err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
7273 		if (err == -ETIMEDOUT)
7274 			break;
7275 
7276 		ocs_value = treq.header.ocs & MASK_OCS;
7277 		if (ocs_value != OCS_SUCCESS) {
7278 			dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
7279 				ocs_value);
7280 			break;
7281 		}
7282 
7283 		memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu));
7284 
7285 		break;
7286 	default:
7287 		err = -EINVAL;
7288 
7289 		break;
7290 	}
7291 
7292 	return err;
7293 }
7294 
7295 /**
7296  * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request
7297  * @hba:	per adapter instance
7298  * @req_upiu:	upiu request
7299  * @rsp_upiu:	upiu reply
7300  * @req_ehs:	EHS field which contains Advanced RPMB Request Message
7301  * @rsp_ehs:	EHS field which returns Advanced RPMB Response Message
7302  * @sg_cnt:	The number of sg lists actually used
7303  * @sg_list:	Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation
7304  * @dir:	DMA direction
7305  *
7306  * Return: zero on success, non-zero on failure.
7307  */
ufshcd_advanced_rpmb_req_handler(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,struct ufs_ehs * req_ehs,struct ufs_ehs * rsp_ehs,int sg_cnt,struct scatterlist * sg_list,enum dma_data_direction dir)7308 int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu,
7309 			 struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs,
7310 			 struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list,
7311 			 enum dma_data_direction dir)
7312 {
7313 	const u32 tag = hba->reserved_slot;
7314 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7315 	int err = 0;
7316 	int result;
7317 	u8 upiu_flags;
7318 	u8 *ehs_data;
7319 	u16 ehs_len;
7320 	int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0;
7321 
7322 	/* Protects use of hba->reserved_slot. */
7323 	ufshcd_dev_man_lock(hba);
7324 
7325 	ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag);
7326 
7327 	ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs);
7328 
7329 	/* update the task tag */
7330 	req_upiu->header.task_tag = tag;
7331 
7332 	/* copy the UPIU(contains CDB) request as it is */
7333 	memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7334 	/* Copy EHS, starting with byte32, immediately after the CDB package */
7335 	memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs));
7336 
7337 	if (dir != DMA_NONE && sg_list)
7338 		ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list);
7339 
7340 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7341 
7342 	err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT);
7343 
7344 	if (!err) {
7345 		/* Just copy the upiu response as it is */
7346 		memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7347 		/* Get the response UPIU result */
7348 		result = (lrbp->ucd_rsp_ptr->header.response << 8) |
7349 			lrbp->ucd_rsp_ptr->header.status;
7350 
7351 		ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length;
7352 		/*
7353 		 * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data
7354 		 * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB
7355 		 * Message is 02h
7356 		 */
7357 		if (ehs_len == 2 && rsp_ehs) {
7358 			/*
7359 			 * ucd_rsp_ptr points to a buffer with a length of 512 bytes
7360 			 * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32
7361 			 */
7362 			ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE;
7363 			memcpy(rsp_ehs, ehs_data, ehs_len * 32);
7364 		}
7365 	}
7366 
7367 	ufshcd_dev_man_unlock(hba);
7368 
7369 	return err ? : result;
7370 }
7371 
7372 /**
7373  * ufshcd_eh_device_reset_handler() - Reset a single logical unit.
7374  * @cmd: SCSI command pointer
7375  *
7376  * Return: SUCCESS or FAILED.
7377  */
ufshcd_eh_device_reset_handler(struct scsi_cmnd * cmd)7378 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7379 {
7380 	unsigned long flags, pending_reqs = 0, not_cleared = 0;
7381 	struct Scsi_Host *host;
7382 	struct ufs_hba *hba;
7383 	struct ufs_hw_queue *hwq;
7384 	struct ufshcd_lrb *lrbp;
7385 	u32 pos, not_cleared_mask = 0;
7386 	int err;
7387 	u8 resp = 0xF, lun;
7388 
7389 	host = cmd->device->host;
7390 	hba = shost_priv(host);
7391 
7392 	lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
7393 	err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
7394 	if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7395 		if (!err)
7396 			err = resp;
7397 		goto out;
7398 	}
7399 
7400 	if (hba->mcq_enabled) {
7401 		for (pos = 0; pos < hba->nutrs; pos++) {
7402 			lrbp = &hba->lrb[pos];
7403 			if (ufshcd_cmd_inflight(lrbp->cmd) &&
7404 			    lrbp->lun == lun) {
7405 				ufshcd_clear_cmd(hba, pos);
7406 				hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
7407 				ufshcd_mcq_poll_cqe_lock(hba, hwq);
7408 			}
7409 		}
7410 		err = 0;
7411 		goto out;
7412 	}
7413 
7414 	/* clear the commands that were pending for corresponding LUN */
7415 	spin_lock_irqsave(&hba->outstanding_lock, flags);
7416 	for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
7417 		if (hba->lrb[pos].lun == lun)
7418 			__set_bit(pos, &pending_reqs);
7419 	hba->outstanding_reqs &= ~pending_reqs;
7420 	spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7421 
7422 	for_each_set_bit(pos, &pending_reqs, hba->nutrs) {
7423 		if (ufshcd_clear_cmd(hba, pos) < 0) {
7424 			spin_lock_irqsave(&hba->outstanding_lock, flags);
7425 			not_cleared = 1U << pos &
7426 				ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7427 			hba->outstanding_reqs |= not_cleared;
7428 			not_cleared_mask |= not_cleared;
7429 			spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7430 
7431 			dev_err(hba->dev, "%s: failed to clear request %d\n",
7432 				__func__, pos);
7433 		}
7434 	}
7435 	__ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask);
7436 
7437 out:
7438 	hba->req_abort_count = 0;
7439 	ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
7440 	if (!err) {
7441 		err = SUCCESS;
7442 	} else {
7443 		dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7444 		err = FAILED;
7445 	}
7446 	return err;
7447 }
7448 
ufshcd_set_req_abort_skip(struct ufs_hba * hba,unsigned long bitmap)7449 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
7450 {
7451 	struct ufshcd_lrb *lrbp;
7452 	int tag;
7453 
7454 	for_each_set_bit(tag, &bitmap, hba->nutrs) {
7455 		lrbp = &hba->lrb[tag];
7456 		lrbp->req_abort_skip = true;
7457 	}
7458 }
7459 
7460 /**
7461  * ufshcd_try_to_abort_task - abort a specific task
7462  * @hba: Pointer to adapter instance
7463  * @tag: Task tag/index to be aborted
7464  *
7465  * Abort the pending command in device by sending UFS_ABORT_TASK task management
7466  * command, and in host controller by clearing the door-bell register. There can
7467  * be race between controller sending the command to the device while abort is
7468  * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
7469  * really issued and then try to abort it.
7470  *
7471  * Return: zero on success, non-zero on failure.
7472  */
ufshcd_try_to_abort_task(struct ufs_hba * hba,int tag)7473 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
7474 {
7475 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7476 	int err = 0;
7477 	int poll_cnt;
7478 	u8 resp = 0xF;
7479 	u32 reg;
7480 
7481 	for (poll_cnt = 100; poll_cnt; poll_cnt--) {
7482 		err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7483 				UFS_QUERY_TASK, &resp);
7484 		if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
7485 			/* cmd pending in the device */
7486 			dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
7487 				__func__, tag);
7488 			break;
7489 		} else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7490 			/*
7491 			 * cmd not pending in the device, check if it is
7492 			 * in transition.
7493 			 */
7494 			dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
7495 				__func__, tag);
7496 			if (hba->mcq_enabled) {
7497 				/* MCQ mode */
7498 				if (ufshcd_cmd_inflight(lrbp->cmd)) {
7499 					/* sleep for max. 200us same delay as in SDB mode */
7500 					usleep_range(100, 200);
7501 					continue;
7502 				}
7503 				/* command completed already */
7504 				dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
7505 					__func__, tag);
7506 				goto out;
7507 			}
7508 
7509 			/* Single Doorbell Mode */
7510 			reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7511 			if (reg & (1 << tag)) {
7512 				/* sleep for max. 200us to stabilize */
7513 				usleep_range(100, 200);
7514 				continue;
7515 			}
7516 			/* command completed already */
7517 			dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
7518 				__func__, tag);
7519 			goto out;
7520 		} else {
7521 			dev_err(hba->dev,
7522 				"%s: no response from device. tag = %d, err %d\n",
7523 				__func__, tag, err);
7524 			if (!err)
7525 				err = resp; /* service response error */
7526 			goto out;
7527 		}
7528 	}
7529 
7530 	if (!poll_cnt) {
7531 		err = -EBUSY;
7532 		goto out;
7533 	}
7534 
7535 	err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7536 			UFS_ABORT_TASK, &resp);
7537 	if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7538 		if (!err) {
7539 			err = resp; /* service response error */
7540 			dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
7541 				__func__, tag, err);
7542 		}
7543 		goto out;
7544 	}
7545 
7546 	err = ufshcd_clear_cmd(hba, tag);
7547 	if (err)
7548 		dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
7549 			__func__, tag, err);
7550 
7551 out:
7552 	return err;
7553 }
7554 
7555 /**
7556  * ufshcd_abort - scsi host template eh_abort_handler callback
7557  * @cmd: SCSI command pointer
7558  *
7559  * Return: SUCCESS or FAILED.
7560  */
ufshcd_abort(struct scsi_cmnd * cmd)7561 static int ufshcd_abort(struct scsi_cmnd *cmd)
7562 {
7563 	struct Scsi_Host *host = cmd->device->host;
7564 	struct ufs_hba *hba = shost_priv(host);
7565 	int tag = scsi_cmd_to_rq(cmd)->tag;
7566 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7567 	unsigned long flags;
7568 	int err = FAILED;
7569 	bool outstanding;
7570 	u32 reg;
7571 
7572 	ufshcd_hold(hba);
7573 
7574 	if (!hba->mcq_enabled) {
7575 		reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7576 		if (!test_bit(tag, &hba->outstanding_reqs)) {
7577 			/* If command is already aborted/completed, return FAILED. */
7578 			dev_err(hba->dev,
7579 				"%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
7580 				__func__, tag, hba->outstanding_reqs, reg);
7581 			goto release;
7582 		}
7583 	}
7584 
7585 	/* Print Transfer Request of aborted task */
7586 	dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
7587 
7588 	/*
7589 	 * Print detailed info about aborted request.
7590 	 * As more than one request might get aborted at the same time,
7591 	 * print full information only for the first aborted request in order
7592 	 * to reduce repeated printouts. For other aborted requests only print
7593 	 * basic details.
7594 	 */
7595 	scsi_print_command(cmd);
7596 	if (!hba->req_abort_count) {
7597 		ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
7598 		ufshcd_print_evt_hist(hba);
7599 		ufshcd_print_host_state(hba);
7600 		ufshcd_print_pwr_info(hba);
7601 		ufshcd_print_tr(hba, tag, true);
7602 	} else {
7603 		ufshcd_print_tr(hba, tag, false);
7604 	}
7605 	hba->req_abort_count++;
7606 
7607 	if (!hba->mcq_enabled && !(reg & (1 << tag))) {
7608 		/* only execute this code in single doorbell mode */
7609 		dev_err(hba->dev,
7610 		"%s: cmd was completed, but without a notifying intr, tag = %d",
7611 		__func__, tag);
7612 		__ufshcd_transfer_req_compl(hba, 1UL << tag);
7613 		goto release;
7614 	}
7615 
7616 	/*
7617 	 * Task abort to the device W-LUN is illegal. When this command
7618 	 * will fail, due to spec violation, scsi err handling next step
7619 	 * will be to send LU reset which, again, is a spec violation.
7620 	 * To avoid these unnecessary/illegal steps, first we clean up
7621 	 * the lrb taken by this cmd and re-set it in outstanding_reqs,
7622 	 * then queue the eh_work and bail.
7623 	 */
7624 	if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
7625 		ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
7626 
7627 		spin_lock_irqsave(host->host_lock, flags);
7628 		hba->force_reset = true;
7629 		ufshcd_schedule_eh_work(hba);
7630 		spin_unlock_irqrestore(host->host_lock, flags);
7631 		goto release;
7632 	}
7633 
7634 	if (hba->mcq_enabled) {
7635 		/* MCQ mode. Branch off to handle abort for mcq mode */
7636 		err = ufshcd_mcq_abort(cmd);
7637 		goto release;
7638 	}
7639 
7640 	/* Skip task abort in case previous aborts failed and report failure */
7641 	if (lrbp->req_abort_skip) {
7642 		dev_err(hba->dev, "%s: skipping abort\n", __func__);
7643 		ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7644 		goto release;
7645 	}
7646 
7647 	err = ufshcd_try_to_abort_task(hba, tag);
7648 	if (err) {
7649 		dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7650 		ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7651 		err = FAILED;
7652 		goto release;
7653 	}
7654 
7655 	/*
7656 	 * Clear the corresponding bit from outstanding_reqs since the command
7657 	 * has been aborted successfully.
7658 	 */
7659 	spin_lock_irqsave(&hba->outstanding_lock, flags);
7660 	outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);
7661 	spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7662 
7663 	if (outstanding)
7664 		ufshcd_release_scsi_cmd(hba, lrbp);
7665 
7666 	err = SUCCESS;
7667 
7668 release:
7669 	/* Matches the ufshcd_hold() call at the start of this function. */
7670 	ufshcd_release(hba);
7671 	return err;
7672 }
7673 
7674 /**
7675  * ufshcd_host_reset_and_restore - reset and restore host controller
7676  * @hba: per-adapter instance
7677  *
7678  * Note that host controller reset may issue DME_RESET to
7679  * local and remote (device) Uni-Pro stack and the attributes
7680  * are reset to default state.
7681  *
7682  * Return: zero on success, non-zero on failure.
7683  */
ufshcd_host_reset_and_restore(struct ufs_hba * hba)7684 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
7685 {
7686 	int err;
7687 
7688 	/*
7689 	 * Stop the host controller and complete the requests
7690 	 * cleared by h/w
7691 	 */
7692 	ufshcd_hba_stop(hba);
7693 	hba->silence_err_logs = true;
7694 	ufshcd_complete_requests(hba, true);
7695 	hba->silence_err_logs = false;
7696 
7697 	/* scale up clocks to max frequency before full reinitialization */
7698 	ufshcd_scale_clks(hba, ULONG_MAX, true);
7699 
7700 	err = ufshcd_hba_enable(hba);
7701 
7702 	/* Establish the link again and restore the device */
7703 	if (!err)
7704 		err = ufshcd_probe_hba(hba, false);
7705 
7706 	if (err)
7707 		dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
7708 	ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
7709 	return err;
7710 }
7711 
7712 /**
7713  * ufshcd_reset_and_restore - reset and re-initialize host/device
7714  * @hba: per-adapter instance
7715  *
7716  * Reset and recover device, host and re-establish link. This
7717  * is helpful to recover the communication in fatal error conditions.
7718  *
7719  * Return: zero on success, non-zero on failure.
7720  */
ufshcd_reset_and_restore(struct ufs_hba * hba)7721 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7722 {
7723 	u32 saved_err = 0;
7724 	u32 saved_uic_err = 0;
7725 	int err = 0;
7726 	unsigned long flags;
7727 	int retries = MAX_HOST_RESET_RETRIES;
7728 
7729 	spin_lock_irqsave(hba->host->host_lock, flags);
7730 	do {
7731 		/*
7732 		 * This is a fresh start, cache and clear saved error first,
7733 		 * in case new error generated during reset and restore.
7734 		 */
7735 		saved_err |= hba->saved_err;
7736 		saved_uic_err |= hba->saved_uic_err;
7737 		hba->saved_err = 0;
7738 		hba->saved_uic_err = 0;
7739 		hba->force_reset = false;
7740 		hba->ufshcd_state = UFSHCD_STATE_RESET;
7741 		spin_unlock_irqrestore(hba->host->host_lock, flags);
7742 
7743 		/* Reset the attached device */
7744 		ufshcd_device_reset(hba);
7745 
7746 		err = ufshcd_host_reset_and_restore(hba);
7747 
7748 		spin_lock_irqsave(hba->host->host_lock, flags);
7749 		if (err)
7750 			continue;
7751 		/* Do not exit unless operational or dead */
7752 		if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
7753 		    hba->ufshcd_state != UFSHCD_STATE_ERROR &&
7754 		    hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL)
7755 			err = -EAGAIN;
7756 	} while (err && --retries);
7757 
7758 	/*
7759 	 * Inform scsi mid-layer that we did reset and allow to handle
7760 	 * Unit Attention properly.
7761 	 */
7762 	scsi_report_bus_reset(hba->host, 0);
7763 	if (err) {
7764 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
7765 		hba->saved_err |= saved_err;
7766 		hba->saved_uic_err |= saved_uic_err;
7767 	}
7768 	spin_unlock_irqrestore(hba->host->host_lock, flags);
7769 
7770 	return err;
7771 }
7772 
7773 /**
7774  * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
7775  * @cmd: SCSI command pointer
7776  *
7777  * Return: SUCCESS or FAILED.
7778  */
ufshcd_eh_host_reset_handler(struct scsi_cmnd * cmd)7779 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7780 {
7781 	int err = SUCCESS;
7782 	unsigned long flags;
7783 	struct ufs_hba *hba;
7784 
7785 	hba = shost_priv(cmd->device->host);
7786 
7787 	/*
7788 	 * If runtime PM sent SSU and got a timeout, scsi_error_handler is
7789 	 * stuck in this function waiting for flush_work(&hba->eh_work). And
7790 	 * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do
7791 	 * ufshcd_link_recovery instead of eh_work to prevent deadlock.
7792 	 */
7793 	if (hba->pm_op_in_progress) {
7794 		if (ufshcd_link_recovery(hba))
7795 			err = FAILED;
7796 
7797 		return err;
7798 	}
7799 
7800 	spin_lock_irqsave(hba->host->host_lock, flags);
7801 	hba->force_reset = true;
7802 	ufshcd_schedule_eh_work(hba);
7803 	dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
7804 	spin_unlock_irqrestore(hba->host->host_lock, flags);
7805 
7806 	flush_work(&hba->eh_work);
7807 
7808 	spin_lock_irqsave(hba->host->host_lock, flags);
7809 	if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
7810 		err = FAILED;
7811 	spin_unlock_irqrestore(hba->host->host_lock, flags);
7812 
7813 	return err;
7814 }
7815 
7816 /**
7817  * ufshcd_get_max_icc_level - calculate the ICC level
7818  * @sup_curr_uA: max. current supported by the regulator
7819  * @start_scan: row at the desc table to start scan from
7820  * @buff: power descriptor buffer
7821  *
7822  * Return: calculated max ICC level for specific regulator.
7823  */
ufshcd_get_max_icc_level(int sup_curr_uA,u32 start_scan,const char * buff)7824 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan,
7825 				    const char *buff)
7826 {
7827 	int i;
7828 	int curr_uA;
7829 	u16 data;
7830 	u16 unit;
7831 
7832 	for (i = start_scan; i >= 0; i--) {
7833 		data = get_unaligned_be16(&buff[2 * i]);
7834 		unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7835 						ATTR_ICC_LVL_UNIT_OFFSET;
7836 		curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7837 		switch (unit) {
7838 		case UFSHCD_NANO_AMP:
7839 			curr_uA = curr_uA / 1000;
7840 			break;
7841 		case UFSHCD_MILI_AMP:
7842 			curr_uA = curr_uA * 1000;
7843 			break;
7844 		case UFSHCD_AMP:
7845 			curr_uA = curr_uA * 1000 * 1000;
7846 			break;
7847 		case UFSHCD_MICRO_AMP:
7848 		default:
7849 			break;
7850 		}
7851 		if (sup_curr_uA >= curr_uA)
7852 			break;
7853 	}
7854 	if (i < 0) {
7855 		i = 0;
7856 		pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7857 	}
7858 
7859 	return (u32)i;
7860 }
7861 
7862 /**
7863  * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level
7864  * In case regulators are not initialized we'll return 0
7865  * @hba: per-adapter instance
7866  * @desc_buf: power descriptor buffer to extract ICC levels from.
7867  *
7868  * Return: calculated ICC level.
7869  */
ufshcd_find_max_sup_active_icc_level(struct ufs_hba * hba,const u8 * desc_buf)7870 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7871 						const u8 *desc_buf)
7872 {
7873 	u32 icc_level = 0;
7874 
7875 	if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7876 						!hba->vreg_info.vccq2) {
7877 		/*
7878 		 * Using dev_dbg to avoid messages during runtime PM to avoid
7879 		 * never-ending cycles of messages written back to storage by
7880 		 * user space causing runtime resume, causing more messages and
7881 		 * so on.
7882 		 */
7883 		dev_dbg(hba->dev,
7884 			"%s: Regulator capability was not set, actvIccLevel=%d",
7885 							__func__, icc_level);
7886 		goto out;
7887 	}
7888 
7889 	if (hba->vreg_info.vcc->max_uA)
7890 		icc_level = ufshcd_get_max_icc_level(
7891 				hba->vreg_info.vcc->max_uA,
7892 				POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7893 				&desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7894 
7895 	if (hba->vreg_info.vccq->max_uA)
7896 		icc_level = ufshcd_get_max_icc_level(
7897 				hba->vreg_info.vccq->max_uA,
7898 				icc_level,
7899 				&desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7900 
7901 	if (hba->vreg_info.vccq2->max_uA)
7902 		icc_level = ufshcd_get_max_icc_level(
7903 				hba->vreg_info.vccq2->max_uA,
7904 				icc_level,
7905 				&desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7906 out:
7907 	return icc_level;
7908 }
7909 
ufshcd_set_active_icc_lvl(struct ufs_hba * hba)7910 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
7911 {
7912 	int ret;
7913 	u8 *desc_buf;
7914 	u32 icc_level;
7915 
7916 	desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
7917 	if (!desc_buf)
7918 		return;
7919 
7920 	ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7921 				     desc_buf, QUERY_DESC_MAX_SIZE);
7922 	if (ret) {
7923 		dev_err(hba->dev,
7924 			"%s: Failed reading power descriptor ret = %d",
7925 			__func__, ret);
7926 		goto out;
7927 	}
7928 
7929 	icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf);
7930 	dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7931 
7932 	ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7933 		QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
7934 
7935 	if (ret)
7936 		dev_err(hba->dev,
7937 			"%s: Failed configuring bActiveICCLevel = %d ret = %d",
7938 			__func__, icc_level, ret);
7939 
7940 out:
7941 	kfree(desc_buf);
7942 }
7943 
ufshcd_blk_pm_runtime_init(struct scsi_device * sdev)7944 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7945 {
7946 	struct Scsi_Host *shost = sdev->host;
7947 
7948 	scsi_autopm_get_device(sdev);
7949 	blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7950 	if (sdev->rpm_autosuspend)
7951 		pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7952 						 shost->rpm_autosuspend_delay);
7953 	scsi_autopm_put_device(sdev);
7954 }
7955 
7956 /**
7957  * ufshcd_scsi_add_wlus - Adds required W-LUs
7958  * @hba: per-adapter instance
7959  *
7960  * UFS device specification requires the UFS devices to support 4 well known
7961  * logical units:
7962  *	"REPORT_LUNS" (address: 01h)
7963  *	"UFS Device" (address: 50h)
7964  *	"RPMB" (address: 44h)
7965  *	"BOOT" (address: 30h)
7966  * UFS device's power management needs to be controlled by "POWER CONDITION"
7967  * field of SSU (START STOP UNIT) command. But this "power condition" field
7968  * will take effect only when its sent to "UFS device" well known logical unit
7969  * hence we require the scsi_device instance to represent this logical unit in
7970  * order for the UFS host driver to send the SSU command for power management.
7971  *
7972  * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7973  * Block) LU so user space process can control this LU. User space may also
7974  * want to have access to BOOT LU.
7975  *
7976  * This function adds scsi device instances for each of all well known LUs
7977  * (except "REPORT LUNS" LU).
7978  *
7979  * Return: zero on success (all required W-LUs are added successfully),
7980  * non-zero error value on failure (if failed to add any of the required W-LU).
7981  */
ufshcd_scsi_add_wlus(struct ufs_hba * hba)7982 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7983 {
7984 	int ret = 0;
7985 	struct scsi_device *sdev_boot, *sdev_rpmb;
7986 
7987 	hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0,
7988 		ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7989 	if (IS_ERR(hba->ufs_device_wlun)) {
7990 		ret = PTR_ERR(hba->ufs_device_wlun);
7991 		hba->ufs_device_wlun = NULL;
7992 		goto out;
7993 	}
7994 	scsi_device_put(hba->ufs_device_wlun);
7995 
7996 	sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
7997 		ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7998 	if (IS_ERR(sdev_rpmb)) {
7999 		ret = PTR_ERR(sdev_rpmb);
8000 		goto remove_ufs_device_wlun;
8001 	}
8002 	ufshcd_blk_pm_runtime_init(sdev_rpmb);
8003 	scsi_device_put(sdev_rpmb);
8004 
8005 	sdev_boot = __scsi_add_device(hba->host, 0, 0,
8006 		ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
8007 	if (IS_ERR(sdev_boot)) {
8008 		dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
8009 	} else {
8010 		ufshcd_blk_pm_runtime_init(sdev_boot);
8011 		scsi_device_put(sdev_boot);
8012 	}
8013 	goto out;
8014 
8015 remove_ufs_device_wlun:
8016 	scsi_remove_device(hba->ufs_device_wlun);
8017 out:
8018 	return ret;
8019 }
8020 
ufshcd_wb_probe(struct ufs_hba * hba,const u8 * desc_buf)8021 static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf)
8022 {
8023 	struct ufs_dev_info *dev_info = &hba->dev_info;
8024 	u8 lun;
8025 	u32 d_lu_wb_buf_alloc;
8026 	u32 ext_ufs_feature;
8027 
8028 	if (!ufshcd_is_wb_allowed(hba))
8029 		return;
8030 
8031 	/*
8032 	 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
8033 	 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
8034 	 * enabled
8035 	 */
8036 	if (!(dev_info->wspecversion >= 0x310 ||
8037 	      dev_info->wspecversion == 0x220 ||
8038 	     (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
8039 		goto wb_disabled;
8040 
8041 	ext_ufs_feature = get_unaligned_be32(desc_buf +
8042 					DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8043 
8044 	if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP))
8045 		goto wb_disabled;
8046 
8047 	/*
8048 	 * WB may be supported but not configured while provisioning. The spec
8049 	 * says, in dedicated wb buffer mode, a max of 1 lun would have wb
8050 	 * buffer configured.
8051 	 */
8052 	dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
8053 
8054 	dev_info->b_presrv_uspc_en =
8055 		desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
8056 
8057 	if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
8058 		if (!get_unaligned_be32(desc_buf +
8059 				   DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS))
8060 			goto wb_disabled;
8061 	} else {
8062 		for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
8063 			d_lu_wb_buf_alloc = 0;
8064 			ufshcd_read_unit_desc_param(hba,
8065 					lun,
8066 					UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
8067 					(u8 *)&d_lu_wb_buf_alloc,
8068 					sizeof(d_lu_wb_buf_alloc));
8069 			if (d_lu_wb_buf_alloc) {
8070 				dev_info->wb_dedicated_lu = lun;
8071 				break;
8072 			}
8073 		}
8074 
8075 		if (!d_lu_wb_buf_alloc)
8076 			goto wb_disabled;
8077 	}
8078 
8079 	if (!ufshcd_is_wb_buf_lifetime_available(hba))
8080 		goto wb_disabled;
8081 
8082 	return;
8083 
8084 wb_disabled:
8085 	hba->caps &= ~UFSHCD_CAP_WB_EN;
8086 }
8087 
ufshcd_temp_notif_probe(struct ufs_hba * hba,const u8 * desc_buf)8088 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf)
8089 {
8090 	struct ufs_dev_info *dev_info = &hba->dev_info;
8091 	u32 ext_ufs_feature;
8092 	u8 mask = 0;
8093 
8094 	if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300)
8095 		return;
8096 
8097 	ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8098 
8099 	if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF)
8100 		mask |= MASK_EE_TOO_LOW_TEMP;
8101 
8102 	if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF)
8103 		mask |= MASK_EE_TOO_HIGH_TEMP;
8104 
8105 	if (mask) {
8106 		ufshcd_enable_ee(hba, mask);
8107 		ufs_hwmon_probe(hba, mask);
8108 	}
8109 }
8110 
ufshcd_ext_iid_probe(struct ufs_hba * hba,u8 * desc_buf)8111 static void ufshcd_ext_iid_probe(struct ufs_hba *hba, u8 *desc_buf)
8112 {
8113 	struct ufs_dev_info *dev_info = &hba->dev_info;
8114 	u32 ext_ufs_feature;
8115 	u32 ext_iid_en = 0;
8116 	int err;
8117 
8118 	/* Only UFS-4.0 and above may support EXT_IID */
8119 	if (dev_info->wspecversion < 0x400)
8120 		goto out;
8121 
8122 	ext_ufs_feature = get_unaligned_be32(desc_buf +
8123 				     DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8124 	if (!(ext_ufs_feature & UFS_DEV_EXT_IID_SUP))
8125 		goto out;
8126 
8127 	err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8128 				      QUERY_ATTR_IDN_EXT_IID_EN, 0, 0, &ext_iid_en);
8129 	if (err)
8130 		dev_err(hba->dev, "failed reading bEXTIIDEn. err = %d\n", err);
8131 
8132 out:
8133 	dev_info->b_ext_iid_en = ext_iid_en;
8134 }
8135 
ufshcd_set_rtt(struct ufs_hba * hba)8136 static void ufshcd_set_rtt(struct ufs_hba *hba)
8137 {
8138 	struct ufs_dev_info *dev_info = &hba->dev_info;
8139 	u32 rtt = 0;
8140 	u32 dev_rtt = 0;
8141 	int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ?
8142 			   hba->vops->max_num_rtt : hba->nortt;
8143 
8144 	/* RTT override makes sense only for UFS-4.0 and above */
8145 	if (dev_info->wspecversion < 0x400)
8146 		return;
8147 
8148 	if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8149 				    QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) {
8150 		dev_err(hba->dev, "failed reading bMaxNumOfRTT\n");
8151 		return;
8152 	}
8153 
8154 	/* do not override if it was already written */
8155 	if (dev_rtt != DEFAULT_MAX_NUM_RTT)
8156 		return;
8157 
8158 	rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap);
8159 
8160 	if (rtt == dev_rtt)
8161 		return;
8162 
8163 	if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8164 				    QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt))
8165 		dev_err(hba->dev, "failed writing bMaxNumOfRTT\n");
8166 }
8167 
ufshcd_fixup_dev_quirks(struct ufs_hba * hba,const struct ufs_dev_quirk * fixups)8168 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba,
8169 			     const struct ufs_dev_quirk *fixups)
8170 {
8171 	const struct ufs_dev_quirk *f;
8172 	struct ufs_dev_info *dev_info = &hba->dev_info;
8173 
8174 	if (!fixups)
8175 		return;
8176 
8177 	for (f = fixups; f->quirk; f++) {
8178 		if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
8179 		     f->wmanufacturerid == UFS_ANY_VENDOR) &&
8180 		     ((dev_info->model &&
8181 		       STR_PRFX_EQUAL(f->model, dev_info->model)) ||
8182 		      !strcmp(f->model, UFS_ANY_MODEL)))
8183 			hba->dev_quirks |= f->quirk;
8184 	}
8185 }
8186 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
8187 
ufs_fixup_device_setup(struct ufs_hba * hba)8188 static void ufs_fixup_device_setup(struct ufs_hba *hba)
8189 {
8190 	/* fix by general quirk table */
8191 	ufshcd_fixup_dev_quirks(hba, ufs_fixups);
8192 
8193 	/* allow vendors to fix quirks */
8194 	ufshcd_vops_fixup_dev_quirks(hba);
8195 }
8196 
ufshcd_update_rtc(struct ufs_hba * hba)8197 static void ufshcd_update_rtc(struct ufs_hba *hba)
8198 {
8199 	struct timespec64 ts64;
8200 	int err;
8201 	u32 val;
8202 
8203 	ktime_get_real_ts64(&ts64);
8204 
8205 	if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) {
8206 		dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__);
8207 		return;
8208 	}
8209 
8210 	/*
8211 	 * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond
8212 	 * 2146 is required, it is recommended to choose the relative RTC mode.
8213 	 */
8214 	val = ts64.tv_sec - hba->dev_info.rtc_time_baseline;
8215 
8216 	/* Skip update RTC if RPM state is not RPM_ACTIVE */
8217 	if (ufshcd_rpm_get_if_active(hba) <= 0)
8218 		return;
8219 
8220 	err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED,
8221 				0, 0, &val);
8222 	ufshcd_rpm_put(hba);
8223 
8224 	if (err)
8225 		dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err);
8226 	else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE)
8227 		hba->dev_info.rtc_time_baseline = ts64.tv_sec;
8228 }
8229 
ufshcd_rtc_work(struct work_struct * work)8230 static void ufshcd_rtc_work(struct work_struct *work)
8231 {
8232 	struct ufs_hba *hba;
8233 
8234 	hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work);
8235 
8236 	 /* Update RTC only when there are no requests in progress and UFSHCI is operational */
8237 	if (!ufshcd_is_ufs_dev_busy(hba) && hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL)
8238 		ufshcd_update_rtc(hba);
8239 
8240 	if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period)
8241 		schedule_delayed_work(&hba->ufs_rtc_update_work,
8242 				      msecs_to_jiffies(hba->dev_info.rtc_update_period));
8243 }
8244 
ufs_init_rtc(struct ufs_hba * hba,u8 * desc_buf)8245 static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf)
8246 {
8247 	u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]);
8248 	struct ufs_dev_info *dev_info = &hba->dev_info;
8249 
8250 	if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) {
8251 		dev_info->rtc_type = UFS_RTC_ABSOLUTE;
8252 
8253 		/*
8254 		 * The concept of measuring time in Linux as the number of seconds elapsed since
8255 		 * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st
8256 		 * 2010 00:00, here we need to adjust ABS baseline.
8257 		 */
8258 		dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) -
8259 							mktime64(1970, 1, 1, 0, 0, 0);
8260 	} else {
8261 		dev_info->rtc_type = UFS_RTC_RELATIVE;
8262 		dev_info->rtc_time_baseline = 0;
8263 	}
8264 
8265 	/*
8266 	 * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state
8267 	 * how to calculate the specific update period for each time unit. And we disable periodic
8268 	 * RTC update work, let user configure by sysfs node according to specific circumstance.
8269 	 */
8270 	dev_info->rtc_update_period = 0;
8271 }
8272 
ufs_get_device_desc(struct ufs_hba * hba)8273 static int ufs_get_device_desc(struct ufs_hba *hba)
8274 {
8275 	int err;
8276 	u8 model_index;
8277 	u8 *desc_buf;
8278 	struct ufs_dev_info *dev_info = &hba->dev_info;
8279 
8280 	desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
8281 	if (!desc_buf) {
8282 		err = -ENOMEM;
8283 		goto out;
8284 	}
8285 
8286 	err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
8287 				     QUERY_DESC_MAX_SIZE);
8288 	if (err) {
8289 		dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
8290 			__func__, err);
8291 		goto out;
8292 	}
8293 
8294 	/*
8295 	 * getting vendor (manufacturerID) and Bank Index in big endian
8296 	 * format
8297 	 */
8298 	dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
8299 				     desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
8300 
8301 	/* getting Specification Version in big endian format */
8302 	dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
8303 				      desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
8304 	dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH];
8305 
8306 	dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP];
8307 
8308 	model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
8309 
8310 	err = ufshcd_read_string_desc(hba, model_index,
8311 				      &dev_info->model, SD_ASCII_STD);
8312 	if (err < 0) {
8313 		dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
8314 			__func__, err);
8315 		goto out;
8316 	}
8317 
8318 	hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
8319 		desc_buf[DEVICE_DESC_PARAM_NUM_WLU];
8320 
8321 	ufs_fixup_device_setup(hba);
8322 
8323 	ufshcd_wb_probe(hba, desc_buf);
8324 
8325 	ufshcd_temp_notif_probe(hba, desc_buf);
8326 
8327 	ufs_init_rtc(hba, desc_buf);
8328 
8329 	if (hba->ext_iid_sup)
8330 		ufshcd_ext_iid_probe(hba, desc_buf);
8331 
8332 	/*
8333 	 * ufshcd_read_string_desc returns size of the string
8334 	 * reset the error value
8335 	 */
8336 	err = 0;
8337 
8338 out:
8339 	kfree(desc_buf);
8340 	return err;
8341 }
8342 
ufs_put_device_desc(struct ufs_hba * hba)8343 static void ufs_put_device_desc(struct ufs_hba *hba)
8344 {
8345 	struct ufs_dev_info *dev_info = &hba->dev_info;
8346 
8347 	kfree(dev_info->model);
8348 	dev_info->model = NULL;
8349 }
8350 
8351 /**
8352  * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
8353  * less than device PA_TACTIVATE time.
8354  * @hba: per-adapter instance
8355  *
8356  * Some UFS devices require host PA_TACTIVATE to be lower than device
8357  * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
8358  * for such devices.
8359  *
8360  * Return: zero on success, non-zero error value on failure.
8361  */
ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba * hba)8362 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
8363 {
8364 	int ret = 0;
8365 	u32 granularity, peer_granularity;
8366 	u32 pa_tactivate, peer_pa_tactivate;
8367 	u32 pa_tactivate_us, peer_pa_tactivate_us;
8368 	static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
8369 
8370 	ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
8371 				  &granularity);
8372 	if (ret)
8373 		goto out;
8374 
8375 	ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
8376 				  &peer_granularity);
8377 	if (ret)
8378 		goto out;
8379 
8380 	if ((granularity < PA_GRANULARITY_MIN_VAL) ||
8381 	    (granularity > PA_GRANULARITY_MAX_VAL)) {
8382 		dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
8383 			__func__, granularity);
8384 		return -EINVAL;
8385 	}
8386 
8387 	if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
8388 	    (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
8389 		dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
8390 			__func__, peer_granularity);
8391 		return -EINVAL;
8392 	}
8393 
8394 	ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
8395 	if (ret)
8396 		goto out;
8397 
8398 	ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
8399 				  &peer_pa_tactivate);
8400 	if (ret)
8401 		goto out;
8402 
8403 	pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
8404 	peer_pa_tactivate_us = peer_pa_tactivate *
8405 			     gran_to_us_table[peer_granularity - 1];
8406 
8407 	if (pa_tactivate_us >= peer_pa_tactivate_us) {
8408 		u32 new_peer_pa_tactivate;
8409 
8410 		new_peer_pa_tactivate = pa_tactivate_us /
8411 				      gran_to_us_table[peer_granularity - 1];
8412 		new_peer_pa_tactivate++;
8413 		ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
8414 					  new_peer_pa_tactivate);
8415 	}
8416 
8417 out:
8418 	return ret;
8419 }
8420 
ufshcd_tune_unipro_params(struct ufs_hba * hba)8421 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
8422 {
8423 	ufshcd_vops_apply_dev_quirks(hba);
8424 
8425 	if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
8426 		/* set 1ms timeout for PA_TACTIVATE */
8427 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
8428 
8429 	if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
8430 		ufshcd_quirk_tune_host_pa_tactivate(hba);
8431 }
8432 
ufshcd_clear_dbg_ufs_stats(struct ufs_hba * hba)8433 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
8434 {
8435 	hba->ufs_stats.hibern8_exit_cnt = 0;
8436 	hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
8437 	hba->req_abort_count = 0;
8438 }
8439 
ufshcd_device_geo_params_init(struct ufs_hba * hba)8440 static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
8441 {
8442 	int err;
8443 	u8 *desc_buf;
8444 
8445 	desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
8446 	if (!desc_buf) {
8447 		err = -ENOMEM;
8448 		goto out;
8449 	}
8450 
8451 	err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
8452 				     desc_buf, QUERY_DESC_MAX_SIZE);
8453 	if (err) {
8454 		dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
8455 				__func__, err);
8456 		goto out;
8457 	}
8458 
8459 	if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
8460 		hba->dev_info.max_lu_supported = 32;
8461 	else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
8462 		hba->dev_info.max_lu_supported = 8;
8463 
8464 out:
8465 	kfree(desc_buf);
8466 	return err;
8467 }
8468 
8469 struct ufs_ref_clk {
8470 	unsigned long freq_hz;
8471 	enum ufs_ref_clk_freq val;
8472 };
8473 
8474 static const struct ufs_ref_clk ufs_ref_clk_freqs[] = {
8475 	{19200000, REF_CLK_FREQ_19_2_MHZ},
8476 	{26000000, REF_CLK_FREQ_26_MHZ},
8477 	{38400000, REF_CLK_FREQ_38_4_MHZ},
8478 	{52000000, REF_CLK_FREQ_52_MHZ},
8479 	{0, REF_CLK_FREQ_INVAL},
8480 };
8481 
8482 static enum ufs_ref_clk_freq
ufs_get_bref_clk_from_hz(unsigned long freq)8483 ufs_get_bref_clk_from_hz(unsigned long freq)
8484 {
8485 	int i;
8486 
8487 	for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
8488 		if (ufs_ref_clk_freqs[i].freq_hz == freq)
8489 			return ufs_ref_clk_freqs[i].val;
8490 
8491 	return REF_CLK_FREQ_INVAL;
8492 }
8493 
ufshcd_parse_dev_ref_clk_freq(struct ufs_hba * hba,struct clk * refclk)8494 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
8495 {
8496 	unsigned long freq;
8497 
8498 	freq = clk_get_rate(refclk);
8499 
8500 	hba->dev_ref_clk_freq =
8501 		ufs_get_bref_clk_from_hz(freq);
8502 
8503 	if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
8504 		dev_err(hba->dev,
8505 		"invalid ref_clk setting = %ld\n", freq);
8506 }
8507 
ufshcd_set_dev_ref_clk(struct ufs_hba * hba)8508 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
8509 {
8510 	int err;
8511 	u32 ref_clk;
8512 	u32 freq = hba->dev_ref_clk_freq;
8513 
8514 	err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8515 			QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
8516 
8517 	if (err) {
8518 		dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
8519 			err);
8520 		goto out;
8521 	}
8522 
8523 	if (ref_clk == freq)
8524 		goto out; /* nothing to update */
8525 
8526 	err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8527 			QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
8528 
8529 	if (err) {
8530 		dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
8531 			ufs_ref_clk_freqs[freq].freq_hz);
8532 		goto out;
8533 	}
8534 
8535 	dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
8536 			ufs_ref_clk_freqs[freq].freq_hz);
8537 
8538 out:
8539 	return err;
8540 }
8541 
ufshcd_device_params_init(struct ufs_hba * hba)8542 static int ufshcd_device_params_init(struct ufs_hba *hba)
8543 {
8544 	bool flag;
8545 	int ret;
8546 
8547 	/* Init UFS geometry descriptor related parameters */
8548 	ret = ufshcd_device_geo_params_init(hba);
8549 	if (ret)
8550 		goto out;
8551 
8552 	/* Check and apply UFS device quirks */
8553 	ret = ufs_get_device_desc(hba);
8554 	if (ret) {
8555 		dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
8556 			__func__, ret);
8557 		goto out;
8558 	}
8559 
8560 	ufshcd_set_rtt(hba);
8561 
8562 	ufshcd_get_ref_clk_gating_wait(hba);
8563 
8564 	if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
8565 			QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
8566 		hba->dev_info.f_power_on_wp_en = flag;
8567 
8568 	/* Probe maximum power mode co-supported by both UFS host and device */
8569 	if (ufshcd_get_max_pwr_mode(hba))
8570 		dev_err(hba->dev,
8571 			"%s: Failed getting max supported power mode\n",
8572 			__func__);
8573 out:
8574 	return ret;
8575 }
8576 
ufshcd_set_timestamp_attr(struct ufs_hba * hba)8577 static void ufshcd_set_timestamp_attr(struct ufs_hba *hba)
8578 {
8579 	int err;
8580 	struct ufs_query_req *request = NULL;
8581 	struct ufs_query_res *response = NULL;
8582 	struct ufs_dev_info *dev_info = &hba->dev_info;
8583 	struct utp_upiu_query_v4_0 *upiu_data;
8584 
8585 	if (dev_info->wspecversion < 0x400)
8586 		return;
8587 
8588 	ufshcd_dev_man_lock(hba);
8589 
8590 	ufshcd_init_query(hba, &request, &response,
8591 			  UPIU_QUERY_OPCODE_WRITE_ATTR,
8592 			  QUERY_ATTR_IDN_TIMESTAMP, 0, 0);
8593 
8594 	request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
8595 
8596 	upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req;
8597 
8598 	put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3);
8599 
8600 	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
8601 
8602 	if (err)
8603 		dev_err(hba->dev, "%s: failed to set timestamp %d\n",
8604 			__func__, err);
8605 
8606 	ufshcd_dev_man_unlock(hba);
8607 }
8608 
8609 /**
8610  * ufshcd_add_lus - probe and add UFS logical units
8611  * @hba: per-adapter instance
8612  *
8613  * Return: 0 upon success; < 0 upon failure.
8614  */
ufshcd_add_lus(struct ufs_hba * hba)8615 static int ufshcd_add_lus(struct ufs_hba *hba)
8616 {
8617 	int ret;
8618 
8619 	/* Add required well known logical units to scsi mid layer */
8620 	ret = ufshcd_scsi_add_wlus(hba);
8621 	if (ret)
8622 		goto out;
8623 
8624 	/* Initialize devfreq after UFS device is detected */
8625 	if (ufshcd_is_clkscaling_supported(hba)) {
8626 		memcpy(&hba->clk_scaling.saved_pwr_info,
8627 			&hba->pwr_info,
8628 			sizeof(struct ufs_pa_layer_attr));
8629 		hba->clk_scaling.is_allowed = true;
8630 
8631 		ret = ufshcd_devfreq_init(hba);
8632 		if (ret)
8633 			goto out;
8634 
8635 		hba->clk_scaling.is_enabled = true;
8636 		ufshcd_init_clk_scaling_sysfs(hba);
8637 	}
8638 
8639 	ufs_bsg_probe(hba);
8640 	scsi_scan_host(hba->host);
8641 
8642 out:
8643 	return ret;
8644 }
8645 
8646 /* SDB - Single Doorbell */
ufshcd_release_sdb_queue(struct ufs_hba * hba,int nutrs)8647 static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs)
8648 {
8649 	size_t ucdl_size, utrdl_size;
8650 
8651 	ucdl_size = ufshcd_get_ucd_size(hba) * nutrs;
8652 	dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr,
8653 			   hba->ucdl_dma_addr);
8654 
8655 	utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs;
8656 	dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr,
8657 			   hba->utrdl_dma_addr);
8658 
8659 	devm_kfree(hba->dev, hba->lrb);
8660 }
8661 
ufshcd_alloc_mcq(struct ufs_hba * hba)8662 static int ufshcd_alloc_mcq(struct ufs_hba *hba)
8663 {
8664 	int ret;
8665 	int old_nutrs = hba->nutrs;
8666 
8667 	ret = ufshcd_mcq_decide_queue_depth(hba);
8668 	if (ret < 0)
8669 		return ret;
8670 
8671 	hba->nutrs = ret;
8672 	ret = ufshcd_mcq_init(hba);
8673 	if (ret)
8674 		goto err;
8675 
8676 	/*
8677 	 * Previously allocated memory for nutrs may not be enough in MCQ mode.
8678 	 * Number of supported tags in MCQ mode may be larger than SDB mode.
8679 	 */
8680 	if (hba->nutrs != old_nutrs) {
8681 		ufshcd_release_sdb_queue(hba, old_nutrs);
8682 		ret = ufshcd_memory_alloc(hba);
8683 		if (ret)
8684 			goto err;
8685 		ufshcd_host_memory_configure(hba);
8686 	}
8687 
8688 	ret = ufshcd_mcq_memory_alloc(hba);
8689 	if (ret)
8690 		goto err;
8691 
8692 	hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
8693 	hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED;
8694 
8695 	return 0;
8696 err:
8697 	hba->nutrs = old_nutrs;
8698 	return ret;
8699 }
8700 
ufshcd_config_mcq(struct ufs_hba * hba)8701 static void ufshcd_config_mcq(struct ufs_hba *hba)
8702 {
8703 	int ret;
8704 	u32 intrs;
8705 
8706 	ret = ufshcd_mcq_vops_config_esi(hba);
8707 	dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : "");
8708 
8709 	intrs = UFSHCD_ENABLE_MCQ_INTRS;
8710 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR)
8711 		intrs &= ~MCQ_CQ_EVENT_STATUS;
8712 	ufshcd_enable_intr(hba, intrs);
8713 	ufshcd_mcq_make_queues_operational(hba);
8714 	ufshcd_mcq_config_mac(hba, hba->nutrs);
8715 
8716 	dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n",
8717 		 hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT],
8718 		 hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL],
8719 		 hba->nutrs);
8720 }
8721 
ufshcd_device_init(struct ufs_hba * hba,bool init_dev_params)8722 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params)
8723 {
8724 	int ret;
8725 	struct Scsi_Host *host = hba->host;
8726 
8727 	hba->ufshcd_state = UFSHCD_STATE_RESET;
8728 
8729 	ret = ufshcd_link_startup(hba);
8730 	if (ret)
8731 		return ret;
8732 
8733 	if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION)
8734 		return ret;
8735 
8736 	/* Debug counters initialization */
8737 	ufshcd_clear_dbg_ufs_stats(hba);
8738 
8739 	/* UniPro link is active now */
8740 	ufshcd_set_link_active(hba);
8741 
8742 	/* Reconfigure MCQ upon reset */
8743 	if (hba->mcq_enabled && !init_dev_params) {
8744 		ufshcd_config_mcq(hba);
8745 		ufshcd_mcq_enable(hba);
8746 	}
8747 
8748 	/* Verify device initialization by sending NOP OUT UPIU */
8749 	ret = ufshcd_verify_dev_init(hba);
8750 	if (ret)
8751 		return ret;
8752 
8753 	/* Initiate UFS initialization, and waiting until completion */
8754 	ret = ufshcd_complete_dev_init(hba);
8755 	if (ret)
8756 		return ret;
8757 
8758 	/*
8759 	 * Initialize UFS device parameters used by driver, these
8760 	 * parameters are associated with UFS descriptors.
8761 	 */
8762 	if (init_dev_params) {
8763 		ret = ufshcd_device_params_init(hba);
8764 		if (ret)
8765 			return ret;
8766 		if (is_mcq_supported(hba) && !hba->scsi_host_added) {
8767 			ufshcd_mcq_enable(hba);
8768 			ret = ufshcd_alloc_mcq(hba);
8769 			if (!ret) {
8770 				ufshcd_config_mcq(hba);
8771 			} else {
8772 				/* Continue with SDB mode */
8773 				ufshcd_mcq_disable(hba);
8774 				use_mcq_mode = false;
8775 				dev_err(hba->dev, "MCQ mode is disabled, err=%d\n",
8776 					 ret);
8777 			}
8778 			ret = scsi_add_host(host, hba->dev);
8779 			if (ret) {
8780 				dev_err(hba->dev, "scsi_add_host failed\n");
8781 				return ret;
8782 			}
8783 			hba->scsi_host_added = true;
8784 		} else if (is_mcq_supported(hba)) {
8785 			/* UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH is set */
8786 			ufshcd_config_mcq(hba);
8787 			ufshcd_mcq_enable(hba);
8788 		}
8789 	}
8790 
8791 	ufshcd_tune_unipro_params(hba);
8792 
8793 	/* UFS device is also active now */
8794 	ufshcd_set_ufs_dev_active(hba);
8795 	ufshcd_force_reset_auto_bkops(hba);
8796 
8797 	ufshcd_set_timestamp_attr(hba);
8798 	schedule_delayed_work(&hba->ufs_rtc_update_work,
8799 			      msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS));
8800 
8801 	/* Gear up to HS gear if supported */
8802 	if (hba->max_pwr_info.is_valid) {
8803 		/*
8804 		 * Set the right value to bRefClkFreq before attempting to
8805 		 * switch to HS gears.
8806 		 */
8807 		if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
8808 			ufshcd_set_dev_ref_clk(hba);
8809 		ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8810 		if (ret) {
8811 			dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
8812 					__func__, ret);
8813 			return ret;
8814 		}
8815 	}
8816 
8817 	return 0;
8818 }
8819 
8820 /**
8821  * ufshcd_probe_hba - probe hba to detect device and initialize it
8822  * @hba: per-adapter instance
8823  * @init_dev_params: whether or not to call ufshcd_device_params_init().
8824  *
8825  * Execute link-startup and verify device initialization
8826  *
8827  * Return: 0 upon success; < 0 upon failure.
8828  */
ufshcd_probe_hba(struct ufs_hba * hba,bool init_dev_params)8829 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params)
8830 {
8831 	ktime_t start = ktime_get();
8832 	unsigned long flags;
8833 	int ret;
8834 
8835 	ret = ufshcd_device_init(hba, init_dev_params);
8836 	if (ret)
8837 		goto out;
8838 
8839 	if (!hba->pm_op_in_progress &&
8840 	    (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) {
8841 		/* Reset the device and controller before doing reinit */
8842 		ufshcd_device_reset(hba);
8843 		ufs_put_device_desc(hba);
8844 		ufshcd_hba_stop(hba);
8845 		ufshcd_vops_reinit_notify(hba);
8846 		ret = ufshcd_hba_enable(hba);
8847 		if (ret) {
8848 			dev_err(hba->dev, "Host controller enable failed\n");
8849 			ufshcd_print_evt_hist(hba);
8850 			ufshcd_print_host_state(hba);
8851 			goto out;
8852 		}
8853 
8854 		/* Reinit the device */
8855 		ret = ufshcd_device_init(hba, init_dev_params);
8856 		if (ret)
8857 			goto out;
8858 	}
8859 
8860 	ufshcd_print_pwr_info(hba);
8861 
8862 	/*
8863 	 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
8864 	 * and for removable UFS card as well, hence always set the parameter.
8865 	 * Note: Error handler may issue the device reset hence resetting
8866 	 * bActiveICCLevel as well so it is always safe to set this here.
8867 	 */
8868 	ufshcd_set_active_icc_lvl(hba);
8869 
8870 	/* Enable UFS Write Booster if supported */
8871 	ufshcd_configure_wb(hba);
8872 
8873 	if (hba->ee_usr_mask)
8874 		ufshcd_write_ee_control(hba);
8875 	ufshcd_configure_auto_hibern8(hba);
8876 
8877 out:
8878 	spin_lock_irqsave(hba->host->host_lock, flags);
8879 	if (ret)
8880 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
8881 	else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
8882 		hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
8883 	spin_unlock_irqrestore(hba->host->host_lock, flags);
8884 
8885 	trace_ufshcd_init(dev_name(hba->dev), ret,
8886 		ktime_to_us(ktime_sub(ktime_get(), start)),
8887 		hba->curr_dev_pwr_mode, hba->uic_link_state);
8888 	return ret;
8889 }
8890 
8891 /**
8892  * ufshcd_async_scan - asynchronous execution for probing hba
8893  * @data: data pointer to pass to this function
8894  * @cookie: cookie data
8895  */
ufshcd_async_scan(void * data,async_cookie_t cookie)8896 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
8897 {
8898 	struct ufs_hba *hba = (struct ufs_hba *)data;
8899 	int ret;
8900 
8901 	down(&hba->host_sem);
8902 	/* Initialize hba, detect and initialize UFS device */
8903 	ret = ufshcd_probe_hba(hba, true);
8904 	up(&hba->host_sem);
8905 	if (ret)
8906 		goto out;
8907 
8908 	/* Probe and add UFS logical units  */
8909 	ret = ufshcd_add_lus(hba);
8910 
8911 out:
8912 	pm_runtime_put_sync(hba->dev);
8913 
8914 	if (ret)
8915 		dev_err(hba->dev, "%s failed: %d\n", __func__, ret);
8916 }
8917 
ufshcd_eh_timed_out(struct scsi_cmnd * scmd)8918 static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
8919 {
8920 	struct ufs_hba *hba = shost_priv(scmd->device->host);
8921 
8922 	if (!hba->system_suspending) {
8923 		/* Activate the error handler in the SCSI core. */
8924 		return SCSI_EH_NOT_HANDLED;
8925 	}
8926 
8927 	/*
8928 	 * If we get here we know that no TMFs are outstanding and also that
8929 	 * the only pending command is a START STOP UNIT command. Handle the
8930 	 * timeout of that command directly to prevent a deadlock between
8931 	 * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler().
8932 	 */
8933 	ufshcd_link_recovery(hba);
8934 	dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n",
8935 		 __func__, hba->outstanding_tasks);
8936 
8937 	return hba->outstanding_reqs ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE;
8938 }
8939 
8940 static const struct attribute_group *ufshcd_driver_groups[] = {
8941 	&ufs_sysfs_unit_descriptor_group,
8942 	&ufs_sysfs_lun_attributes_group,
8943 	NULL,
8944 };
8945 
8946 static struct ufs_hba_variant_params ufs_hba_vps = {
8947 	.hba_enable_delay_us		= 1000,
8948 	.wb_flush_threshold		= UFS_WB_BUF_REMAIN_PERCENT(40),
8949 	.devfreq_profile.polling_ms	= 100,
8950 	.devfreq_profile.target		= ufshcd_devfreq_target,
8951 	.devfreq_profile.get_dev_status	= ufshcd_devfreq_get_dev_status,
8952 	.ondemand_data.upthreshold	= 70,
8953 	.ondemand_data.downdifferential	= 5,
8954 };
8955 
8956 static const struct scsi_host_template ufshcd_driver_template = {
8957 	.module			= THIS_MODULE,
8958 	.name			= UFSHCD,
8959 	.proc_name		= UFSHCD,
8960 	.map_queues		= ufshcd_map_queues,
8961 	.queuecommand		= ufshcd_queuecommand,
8962 	.mq_poll		= ufshcd_poll,
8963 	.slave_alloc		= ufshcd_slave_alloc,
8964 	.device_configure	= ufshcd_device_configure,
8965 	.slave_destroy		= ufshcd_slave_destroy,
8966 	.change_queue_depth	= ufshcd_change_queue_depth,
8967 	.eh_abort_handler	= ufshcd_abort,
8968 	.eh_device_reset_handler = ufshcd_eh_device_reset_handler,
8969 	.eh_host_reset_handler   = ufshcd_eh_host_reset_handler,
8970 	.eh_timed_out		= ufshcd_eh_timed_out,
8971 	.this_id		= -1,
8972 	.sg_tablesize		= SG_ALL,
8973 	.max_segment_size	= PRDT_DATA_BYTE_COUNT_MAX,
8974 	.max_sectors		= SZ_1M / SECTOR_SIZE,
8975 	.max_host_blocked	= 1,
8976 	.track_queue_depth	= 1,
8977 	.skip_settle_delay	= 1,
8978 	.sdev_groups		= ufshcd_driver_groups,
8979 };
8980 
ufshcd_config_vreg_load(struct device * dev,struct ufs_vreg * vreg,int ua)8981 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
8982 				   int ua)
8983 {
8984 	int ret;
8985 
8986 	if (!vreg)
8987 		return 0;
8988 
8989 	/*
8990 	 * "set_load" operation shall be required on those regulators
8991 	 * which specifically configured current limitation. Otherwise
8992 	 * zero max_uA may cause unexpected behavior when regulator is
8993 	 * enabled or set as high power mode.
8994 	 */
8995 	if (!vreg->max_uA)
8996 		return 0;
8997 
8998 	ret = regulator_set_load(vreg->reg, ua);
8999 	if (ret < 0) {
9000 		dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
9001 				__func__, vreg->name, ua, ret);
9002 	}
9003 
9004 	return ret;
9005 }
9006 
ufshcd_config_vreg_lpm(struct ufs_hba * hba,struct ufs_vreg * vreg)9007 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
9008 					 struct ufs_vreg *vreg)
9009 {
9010 	return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
9011 }
9012 
ufshcd_config_vreg_hpm(struct ufs_hba * hba,struct ufs_vreg * vreg)9013 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
9014 					 struct ufs_vreg *vreg)
9015 {
9016 	if (!vreg)
9017 		return 0;
9018 
9019 	return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
9020 }
9021 
ufshcd_config_vreg(struct device * dev,struct ufs_vreg * vreg,bool on)9022 static int ufshcd_config_vreg(struct device *dev,
9023 		struct ufs_vreg *vreg, bool on)
9024 {
9025 	if (regulator_count_voltages(vreg->reg) <= 0)
9026 		return 0;
9027 
9028 	return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0);
9029 }
9030 
ufshcd_enable_vreg(struct device * dev,struct ufs_vreg * vreg)9031 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
9032 {
9033 	int ret = 0;
9034 
9035 	if (!vreg || vreg->enabled)
9036 		goto out;
9037 
9038 	ret = ufshcd_config_vreg(dev, vreg, true);
9039 	if (!ret)
9040 		ret = regulator_enable(vreg->reg);
9041 
9042 	if (!ret)
9043 		vreg->enabled = true;
9044 	else
9045 		dev_err(dev, "%s: %s enable failed, err=%d\n",
9046 				__func__, vreg->name, ret);
9047 out:
9048 	return ret;
9049 }
9050 
ufshcd_disable_vreg(struct device * dev,struct ufs_vreg * vreg)9051 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
9052 {
9053 	int ret = 0;
9054 
9055 	if (!vreg || !vreg->enabled || vreg->always_on)
9056 		goto out;
9057 
9058 	ret = regulator_disable(vreg->reg);
9059 
9060 	if (!ret) {
9061 		/* ignore errors on applying disable config */
9062 		ufshcd_config_vreg(dev, vreg, false);
9063 		vreg->enabled = false;
9064 	} else {
9065 		dev_err(dev, "%s: %s disable failed, err=%d\n",
9066 				__func__, vreg->name, ret);
9067 	}
9068 out:
9069 	return ret;
9070 }
9071 
ufshcd_setup_vreg(struct ufs_hba * hba,bool on)9072 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
9073 {
9074 	int ret = 0;
9075 	struct device *dev = hba->dev;
9076 	struct ufs_vreg_info *info = &hba->vreg_info;
9077 
9078 	ret = ufshcd_toggle_vreg(dev, info->vcc, on);
9079 	if (ret)
9080 		goto out;
9081 
9082 	ret = ufshcd_toggle_vreg(dev, info->vccq, on);
9083 	if (ret)
9084 		goto out;
9085 
9086 	ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
9087 
9088 out:
9089 	if (ret) {
9090 		ufshcd_toggle_vreg(dev, info->vccq2, false);
9091 		ufshcd_toggle_vreg(dev, info->vccq, false);
9092 		ufshcd_toggle_vreg(dev, info->vcc, false);
9093 	}
9094 	return ret;
9095 }
9096 
ufshcd_setup_hba_vreg(struct ufs_hba * hba,bool on)9097 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
9098 {
9099 	struct ufs_vreg_info *info = &hba->vreg_info;
9100 
9101 	return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
9102 }
9103 
ufshcd_get_vreg(struct device * dev,struct ufs_vreg * vreg)9104 int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
9105 {
9106 	int ret = 0;
9107 
9108 	if (!vreg)
9109 		goto out;
9110 
9111 	vreg->reg = devm_regulator_get(dev, vreg->name);
9112 	if (IS_ERR(vreg->reg)) {
9113 		ret = PTR_ERR(vreg->reg);
9114 		dev_err(dev, "%s: %s get failed, err=%d\n",
9115 				__func__, vreg->name, ret);
9116 	}
9117 out:
9118 	return ret;
9119 }
9120 EXPORT_SYMBOL_GPL(ufshcd_get_vreg);
9121 
ufshcd_init_vreg(struct ufs_hba * hba)9122 static int ufshcd_init_vreg(struct ufs_hba *hba)
9123 {
9124 	int ret = 0;
9125 	struct device *dev = hba->dev;
9126 	struct ufs_vreg_info *info = &hba->vreg_info;
9127 
9128 	ret = ufshcd_get_vreg(dev, info->vcc);
9129 	if (ret)
9130 		goto out;
9131 
9132 	ret = ufshcd_get_vreg(dev, info->vccq);
9133 	if (!ret)
9134 		ret = ufshcd_get_vreg(dev, info->vccq2);
9135 out:
9136 	return ret;
9137 }
9138 
ufshcd_init_hba_vreg(struct ufs_hba * hba)9139 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
9140 {
9141 	struct ufs_vreg_info *info = &hba->vreg_info;
9142 
9143 	return ufshcd_get_vreg(hba->dev, info->vdd_hba);
9144 }
9145 
ufshcd_setup_clocks(struct ufs_hba * hba,bool on)9146 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
9147 {
9148 	int ret = 0;
9149 	struct ufs_clk_info *clki;
9150 	struct list_head *head = &hba->clk_list_head;
9151 	unsigned long flags;
9152 	ktime_t start = ktime_get();
9153 	bool clk_state_changed = false;
9154 
9155 	if (list_empty(head))
9156 		goto out;
9157 
9158 	ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
9159 	if (ret)
9160 		return ret;
9161 
9162 	list_for_each_entry(clki, head, list) {
9163 		if (!IS_ERR_OR_NULL(clki->clk)) {
9164 			/*
9165 			 * Don't disable clocks which are needed
9166 			 * to keep the link active.
9167 			 */
9168 			if (ufshcd_is_link_active(hba) &&
9169 			    clki->keep_link_active)
9170 				continue;
9171 
9172 			clk_state_changed = on ^ clki->enabled;
9173 			if (on && !clki->enabled) {
9174 				ret = clk_prepare_enable(clki->clk);
9175 				if (ret) {
9176 					dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
9177 						__func__, clki->name, ret);
9178 					goto out;
9179 				}
9180 			} else if (!on && clki->enabled) {
9181 				clk_disable_unprepare(clki->clk);
9182 			}
9183 			clki->enabled = on;
9184 			dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
9185 					clki->name, on ? "en" : "dis");
9186 		}
9187 	}
9188 
9189 	ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
9190 	if (ret)
9191 		return ret;
9192 
9193 	if (!ufshcd_is_clkscaling_supported(hba))
9194 		ufshcd_pm_qos_update(hba, on);
9195 out:
9196 	if (ret) {
9197 		list_for_each_entry(clki, head, list) {
9198 			if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
9199 				clk_disable_unprepare(clki->clk);
9200 		}
9201 	} else if (!ret && on) {
9202 		spin_lock_irqsave(hba->host->host_lock, flags);
9203 		hba->clk_gating.state = CLKS_ON;
9204 		trace_ufshcd_clk_gating(dev_name(hba->dev),
9205 					hba->clk_gating.state);
9206 		spin_unlock_irqrestore(hba->host->host_lock, flags);
9207 	}
9208 
9209 	if (clk_state_changed)
9210 		trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
9211 			(on ? "on" : "off"),
9212 			ktime_to_us(ktime_sub(ktime_get(), start)), ret);
9213 	return ret;
9214 }
9215 
ufshcd_parse_ref_clk_property(struct ufs_hba * hba)9216 static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba)
9217 {
9218 	u32 freq;
9219 	int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq);
9220 
9221 	if (ret) {
9222 		dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret);
9223 		return REF_CLK_FREQ_INVAL;
9224 	}
9225 
9226 	return ufs_get_bref_clk_from_hz(freq);
9227 }
9228 
ufshcd_init_clocks(struct ufs_hba * hba)9229 static int ufshcd_init_clocks(struct ufs_hba *hba)
9230 {
9231 	int ret = 0;
9232 	struct ufs_clk_info *clki;
9233 	struct device *dev = hba->dev;
9234 	struct list_head *head = &hba->clk_list_head;
9235 
9236 	if (list_empty(head))
9237 		goto out;
9238 
9239 	list_for_each_entry(clki, head, list) {
9240 		if (!clki->name)
9241 			continue;
9242 
9243 		clki->clk = devm_clk_get(dev, clki->name);
9244 		if (IS_ERR(clki->clk)) {
9245 			ret = PTR_ERR(clki->clk);
9246 			dev_err(dev, "%s: %s clk get failed, %d\n",
9247 					__func__, clki->name, ret);
9248 			goto out;
9249 		}
9250 
9251 		/*
9252 		 * Parse device ref clk freq as per device tree "ref_clk".
9253 		 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
9254 		 * in ufshcd_alloc_host().
9255 		 */
9256 		if (!strcmp(clki->name, "ref_clk"))
9257 			ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
9258 
9259 		if (clki->max_freq) {
9260 			ret = clk_set_rate(clki->clk, clki->max_freq);
9261 			if (ret) {
9262 				dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
9263 					__func__, clki->name,
9264 					clki->max_freq, ret);
9265 				goto out;
9266 			}
9267 			clki->curr_freq = clki->max_freq;
9268 		}
9269 		dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
9270 				clki->name, clk_get_rate(clki->clk));
9271 	}
9272 
9273 	/* Set Max. frequency for all clocks */
9274 	if (hba->use_pm_opp) {
9275 		ret = ufshcd_opp_set_rate(hba, ULONG_MAX);
9276 		if (ret) {
9277 			dev_err(hba->dev, "%s: failed to set OPP: %d", __func__,
9278 				ret);
9279 			goto out;
9280 		}
9281 	}
9282 
9283 out:
9284 	return ret;
9285 }
9286 
ufshcd_variant_hba_init(struct ufs_hba * hba)9287 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
9288 {
9289 	int err = 0;
9290 
9291 	if (!hba->vops)
9292 		goto out;
9293 
9294 	err = ufshcd_vops_init(hba);
9295 	if (err)
9296 		dev_err_probe(hba->dev, err,
9297 			      "%s: variant %s init failed with err %d\n",
9298 			      __func__, ufshcd_get_var_name(hba), err);
9299 out:
9300 	return err;
9301 }
9302 
ufshcd_variant_hba_exit(struct ufs_hba * hba)9303 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
9304 {
9305 	if (!hba->vops)
9306 		return;
9307 
9308 	ufshcd_vops_exit(hba);
9309 }
9310 
ufshcd_hba_init(struct ufs_hba * hba)9311 static int ufshcd_hba_init(struct ufs_hba *hba)
9312 {
9313 	int err;
9314 
9315 	/*
9316 	 * Handle host controller power separately from the UFS device power
9317 	 * rails as it will help controlling the UFS host controller power
9318 	 * collapse easily which is different than UFS device power collapse.
9319 	 * Also, enable the host controller power before we go ahead with rest
9320 	 * of the initialization here.
9321 	 */
9322 	err = ufshcd_init_hba_vreg(hba);
9323 	if (err)
9324 		goto out;
9325 
9326 	err = ufshcd_setup_hba_vreg(hba, true);
9327 	if (err)
9328 		goto out;
9329 
9330 	err = ufshcd_init_clocks(hba);
9331 	if (err)
9332 		goto out_disable_hba_vreg;
9333 
9334 	if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
9335 		hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba);
9336 
9337 	err = ufshcd_setup_clocks(hba, true);
9338 	if (err)
9339 		goto out_disable_hba_vreg;
9340 
9341 	err = ufshcd_init_vreg(hba);
9342 	if (err)
9343 		goto out_disable_clks;
9344 
9345 	err = ufshcd_setup_vreg(hba, true);
9346 	if (err)
9347 		goto out_disable_clks;
9348 
9349 	err = ufshcd_variant_hba_init(hba);
9350 	if (err)
9351 		goto out_disable_vreg;
9352 
9353 	ufs_debugfs_hba_init(hba);
9354 	ufs_fault_inject_hba_init(hba);
9355 
9356 	hba->is_powered = true;
9357 	goto out;
9358 
9359 out_disable_vreg:
9360 	ufshcd_setup_vreg(hba, false);
9361 out_disable_clks:
9362 	ufshcd_setup_clocks(hba, false);
9363 out_disable_hba_vreg:
9364 	ufshcd_setup_hba_vreg(hba, false);
9365 out:
9366 	return err;
9367 }
9368 
ufshcd_hba_exit(struct ufs_hba * hba)9369 static void ufshcd_hba_exit(struct ufs_hba *hba)
9370 {
9371 	if (hba->is_powered) {
9372 		ufshcd_pm_qos_exit(hba);
9373 		ufshcd_exit_clk_scaling(hba);
9374 		ufshcd_exit_clk_gating(hba);
9375 		if (hba->eh_wq)
9376 			destroy_workqueue(hba->eh_wq);
9377 		ufs_debugfs_hba_exit(hba);
9378 		ufshcd_variant_hba_exit(hba);
9379 		ufshcd_setup_vreg(hba, false);
9380 		ufshcd_setup_clocks(hba, false);
9381 		ufshcd_setup_hba_vreg(hba, false);
9382 		hba->is_powered = false;
9383 		ufs_put_device_desc(hba);
9384 	}
9385 }
9386 
ufshcd_execute_start_stop(struct scsi_device * sdev,enum ufs_dev_pwr_mode pwr_mode,struct scsi_sense_hdr * sshdr)9387 static int ufshcd_execute_start_stop(struct scsi_device *sdev,
9388 				     enum ufs_dev_pwr_mode pwr_mode,
9389 				     struct scsi_sense_hdr *sshdr)
9390 {
9391 	const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 };
9392 	struct scsi_failure failure_defs[] = {
9393 		{
9394 			.allowed = 2,
9395 			.result = SCMD_FAILURE_RESULT_ANY,
9396 		},
9397 	};
9398 	struct scsi_failures failures = {
9399 		.failure_definitions = failure_defs,
9400 	};
9401 	const struct scsi_exec_args args = {
9402 		.failures = &failures,
9403 		.sshdr = sshdr,
9404 		.req_flags = BLK_MQ_REQ_PM,
9405 		.scmd_flags = SCMD_FAIL_IF_RECOVERING,
9406 	};
9407 
9408 	return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL,
9409 			/*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0,
9410 			&args);
9411 }
9412 
9413 /**
9414  * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
9415  *			     power mode
9416  * @hba: per adapter instance
9417  * @pwr_mode: device power mode to set
9418  *
9419  * Return: 0 if requested power mode is set successfully;
9420  *         < 0 if failed to set the requested power mode.
9421  */
ufshcd_set_dev_pwr_mode(struct ufs_hba * hba,enum ufs_dev_pwr_mode pwr_mode)9422 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
9423 				     enum ufs_dev_pwr_mode pwr_mode)
9424 {
9425 	struct scsi_sense_hdr sshdr;
9426 	struct scsi_device *sdp;
9427 	unsigned long flags;
9428 	int ret;
9429 
9430 	spin_lock_irqsave(hba->host->host_lock, flags);
9431 	sdp = hba->ufs_device_wlun;
9432 	if (sdp && scsi_device_online(sdp))
9433 		ret = scsi_device_get(sdp);
9434 	else
9435 		ret = -ENODEV;
9436 	spin_unlock_irqrestore(hba->host->host_lock, flags);
9437 
9438 	if (ret)
9439 		return ret;
9440 
9441 	/*
9442 	 * If scsi commands fail, the scsi mid-layer schedules scsi error-
9443 	 * handling, which would wait for host to be resumed. Since we know
9444 	 * we are functional while we are here, skip host resume in error
9445 	 * handling context.
9446 	 */
9447 	hba->host->eh_noresume = 1;
9448 
9449 	/*
9450 	 * Current function would be generally called from the power management
9451 	 * callbacks hence set the RQF_PM flag so that it doesn't resume the
9452 	 * already suspended childs.
9453 	 */
9454 	ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr);
9455 	if (ret) {
9456 		sdev_printk(KERN_WARNING, sdp,
9457 			    "START_STOP failed for power mode: %d, result %x\n",
9458 			    pwr_mode, ret);
9459 		if (ret > 0) {
9460 			if (scsi_sense_valid(&sshdr))
9461 				scsi_print_sense_hdr(sdp, NULL, &sshdr);
9462 			ret = -EIO;
9463 		}
9464 	} else {
9465 		hba->curr_dev_pwr_mode = pwr_mode;
9466 	}
9467 
9468 	scsi_device_put(sdp);
9469 	hba->host->eh_noresume = 0;
9470 	return ret;
9471 }
9472 
ufshcd_link_state_transition(struct ufs_hba * hba,enum uic_link_state req_link_state,bool check_for_bkops)9473 static int ufshcd_link_state_transition(struct ufs_hba *hba,
9474 					enum uic_link_state req_link_state,
9475 					bool check_for_bkops)
9476 {
9477 	int ret = 0;
9478 
9479 	if (req_link_state == hba->uic_link_state)
9480 		return 0;
9481 
9482 	if (req_link_state == UIC_LINK_HIBERN8_STATE) {
9483 		ret = ufshcd_uic_hibern8_enter(hba);
9484 		if (!ret) {
9485 			ufshcd_set_link_hibern8(hba);
9486 		} else {
9487 			dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9488 					__func__, ret);
9489 			goto out;
9490 		}
9491 	}
9492 	/*
9493 	 * If autobkops is enabled, link can't be turned off because
9494 	 * turning off the link would also turn off the device, except in the
9495 	 * case of DeepSleep where the device is expected to remain powered.
9496 	 */
9497 	else if ((req_link_state == UIC_LINK_OFF_STATE) &&
9498 		 (!check_for_bkops || !hba->auto_bkops_enabled)) {
9499 		/*
9500 		 * Let's make sure that link is in low power mode, we are doing
9501 		 * this currently by putting the link in Hibern8. Otherway to
9502 		 * put the link in low power mode is to send the DME end point
9503 		 * to device and then send the DME reset command to local
9504 		 * unipro. But putting the link in hibern8 is much faster.
9505 		 *
9506 		 * Note also that putting the link in Hibern8 is a requirement
9507 		 * for entering DeepSleep.
9508 		 */
9509 		ret = ufshcd_uic_hibern8_enter(hba);
9510 		if (ret) {
9511 			dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9512 					__func__, ret);
9513 			goto out;
9514 		}
9515 		/*
9516 		 * Change controller state to "reset state" which
9517 		 * should also put the link in off/reset state
9518 		 */
9519 		ufshcd_hba_stop(hba);
9520 		/*
9521 		 * TODO: Check if we need any delay to make sure that
9522 		 * controller is reset
9523 		 */
9524 		ufshcd_set_link_off(hba);
9525 	}
9526 
9527 out:
9528 	return ret;
9529 }
9530 
ufshcd_vreg_set_lpm(struct ufs_hba * hba)9531 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
9532 {
9533 	bool vcc_off = false;
9534 
9535 	/*
9536 	 * It seems some UFS devices may keep drawing more than sleep current
9537 	 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
9538 	 * To avoid this situation, add 2ms delay before putting these UFS
9539 	 * rails in LPM mode.
9540 	 */
9541 	if (!ufshcd_is_link_active(hba) &&
9542 	    hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
9543 		usleep_range(2000, 2100);
9544 
9545 	/*
9546 	 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
9547 	 * power.
9548 	 *
9549 	 * If UFS device and link is in OFF state, all power supplies (VCC,
9550 	 * VCCQ, VCCQ2) can be turned off if power on write protect is not
9551 	 * required. If UFS link is inactive (Hibern8 or OFF state) and device
9552 	 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
9553 	 *
9554 	 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
9555 	 * in low power state which would save some power.
9556 	 *
9557 	 * If Write Booster is enabled and the device needs to flush the WB
9558 	 * buffer OR if bkops status is urgent for WB, keep Vcc on.
9559 	 */
9560 	if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
9561 	    !hba->dev_info.is_lu_power_on_wp) {
9562 		ufshcd_setup_vreg(hba, false);
9563 		vcc_off = true;
9564 	} else if (!ufshcd_is_ufs_dev_active(hba)) {
9565 		ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9566 		vcc_off = true;
9567 		if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) {
9568 			ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9569 			ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
9570 		}
9571 	}
9572 
9573 	/*
9574 	 * Some UFS devices require delay after VCC power rail is turned-off.
9575 	 */
9576 	if (vcc_off && hba->vreg_info.vcc &&
9577 		hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
9578 		usleep_range(5000, 5100);
9579 }
9580 
9581 #ifdef CONFIG_PM
ufshcd_vreg_set_hpm(struct ufs_hba * hba)9582 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
9583 {
9584 	int ret = 0;
9585 
9586 	if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
9587 	    !hba->dev_info.is_lu_power_on_wp) {
9588 		ret = ufshcd_setup_vreg(hba, true);
9589 	} else if (!ufshcd_is_ufs_dev_active(hba)) {
9590 		if (!ufshcd_is_link_active(hba)) {
9591 			ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
9592 			if (ret)
9593 				goto vcc_disable;
9594 			ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
9595 			if (ret)
9596 				goto vccq_lpm;
9597 		}
9598 		ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
9599 	}
9600 	goto out;
9601 
9602 vccq_lpm:
9603 	ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9604 vcc_disable:
9605 	ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9606 out:
9607 	return ret;
9608 }
9609 #endif /* CONFIG_PM */
9610 
ufshcd_hba_vreg_set_lpm(struct ufs_hba * hba)9611 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
9612 {
9613 	if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
9614 		ufshcd_setup_hba_vreg(hba, false);
9615 }
9616 
ufshcd_hba_vreg_set_hpm(struct ufs_hba * hba)9617 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
9618 {
9619 	if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
9620 		ufshcd_setup_hba_vreg(hba, true);
9621 }
9622 
__ufshcd_wl_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op)9623 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9624 {
9625 	int ret = 0;
9626 	bool check_for_bkops;
9627 	enum ufs_pm_level pm_lvl;
9628 	enum ufs_dev_pwr_mode req_dev_pwr_mode;
9629 	enum uic_link_state req_link_state;
9630 
9631 	hba->pm_op_in_progress = true;
9632 	if (pm_op != UFS_SHUTDOWN_PM) {
9633 		pm_lvl = pm_op == UFS_RUNTIME_PM ?
9634 			 hba->rpm_lvl : hba->spm_lvl;
9635 		req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
9636 		req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
9637 	} else {
9638 		req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
9639 		req_link_state = UIC_LINK_OFF_STATE;
9640 	}
9641 
9642 	/*
9643 	 * If we can't transition into any of the low power modes
9644 	 * just gate the clocks.
9645 	 */
9646 	ufshcd_hold(hba);
9647 	hba->clk_gating.is_suspended = true;
9648 
9649 	if (ufshcd_is_clkscaling_supported(hba))
9650 		ufshcd_clk_scaling_suspend(hba, true);
9651 
9652 	if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
9653 			req_link_state == UIC_LINK_ACTIVE_STATE) {
9654 		goto vops_suspend;
9655 	}
9656 
9657 	if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
9658 	    (req_link_state == hba->uic_link_state))
9659 		goto enable_scaling;
9660 
9661 	/* UFS device & link must be active before we enter in this function */
9662 	if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
9663 		/*  Wait err handler finish or trigger err recovery */
9664 		if (!ufshcd_eh_in_progress(hba))
9665 			ufshcd_force_error_recovery(hba);
9666 		ret = -EBUSY;
9667 		goto enable_scaling;
9668 	}
9669 
9670 	if (pm_op == UFS_RUNTIME_PM) {
9671 		if (ufshcd_can_autobkops_during_suspend(hba)) {
9672 			/*
9673 			 * The device is idle with no requests in the queue,
9674 			 * allow background operations if bkops status shows
9675 			 * that performance might be impacted.
9676 			 */
9677 			ret = ufshcd_bkops_ctrl(hba);
9678 			if (ret) {
9679 				/*
9680 				 * If return err in suspend flow, IO will hang.
9681 				 * Trigger error handler and break suspend for
9682 				 * error recovery.
9683 				 */
9684 				ufshcd_force_error_recovery(hba);
9685 				ret = -EBUSY;
9686 				goto enable_scaling;
9687 			}
9688 		} else {
9689 			/* make sure that auto bkops is disabled */
9690 			ufshcd_disable_auto_bkops(hba);
9691 		}
9692 		/*
9693 		 * If device needs to do BKOP or WB buffer flush during
9694 		 * Hibern8, keep device power mode as "active power mode"
9695 		 * and VCC supply.
9696 		 */
9697 		hba->dev_info.b_rpm_dev_flush_capable =
9698 			hba->auto_bkops_enabled ||
9699 			(((req_link_state == UIC_LINK_HIBERN8_STATE) ||
9700 			((req_link_state == UIC_LINK_ACTIVE_STATE) &&
9701 			ufshcd_is_auto_hibern8_enabled(hba))) &&
9702 			ufshcd_wb_need_flush(hba));
9703 	}
9704 
9705 	flush_work(&hba->eeh_work);
9706 
9707 	ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9708 	if (ret)
9709 		goto enable_scaling;
9710 
9711 	if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
9712 		if (pm_op != UFS_RUNTIME_PM)
9713 			/* ensure that bkops is disabled */
9714 			ufshcd_disable_auto_bkops(hba);
9715 
9716 		if (!hba->dev_info.b_rpm_dev_flush_capable) {
9717 			ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
9718 			if (ret && pm_op != UFS_SHUTDOWN_PM) {
9719 				/*
9720 				 * If return err in suspend flow, IO will hang.
9721 				 * Trigger error handler and break suspend for
9722 				 * error recovery.
9723 				 */
9724 				ufshcd_force_error_recovery(hba);
9725 				ret = -EBUSY;
9726 			}
9727 			if (ret)
9728 				goto enable_scaling;
9729 		}
9730 	}
9731 
9732 	/*
9733 	 * In the case of DeepSleep, the device is expected to remain powered
9734 	 * with the link off, so do not check for bkops.
9735 	 */
9736 	check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
9737 	ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
9738 	if (ret && pm_op != UFS_SHUTDOWN_PM) {
9739 		/*
9740 		 * If return err in suspend flow, IO will hang.
9741 		 * Trigger error handler and break suspend for
9742 		 * error recovery.
9743 		 */
9744 		ufshcd_force_error_recovery(hba);
9745 		ret = -EBUSY;
9746 	}
9747 	if (ret)
9748 		goto set_dev_active;
9749 
9750 vops_suspend:
9751 	/*
9752 	 * Call vendor specific suspend callback. As these callbacks may access
9753 	 * vendor specific host controller register space call them before the
9754 	 * host clocks are ON.
9755 	 */
9756 	ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9757 	if (ret)
9758 		goto set_link_active;
9759 
9760 	cancel_delayed_work_sync(&hba->ufs_rtc_update_work);
9761 	goto out;
9762 
9763 set_link_active:
9764 	/*
9765 	 * Device hardware reset is required to exit DeepSleep. Also, for
9766 	 * DeepSleep, the link is off so host reset and restore will be done
9767 	 * further below.
9768 	 */
9769 	if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9770 		ufshcd_device_reset(hba);
9771 		WARN_ON(!ufshcd_is_link_off(hba));
9772 	}
9773 	if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
9774 		ufshcd_set_link_active(hba);
9775 	else if (ufshcd_is_link_off(hba))
9776 		ufshcd_host_reset_and_restore(hba);
9777 set_dev_active:
9778 	/* Can also get here needing to exit DeepSleep */
9779 	if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9780 		ufshcd_device_reset(hba);
9781 		ufshcd_host_reset_and_restore(hba);
9782 	}
9783 	if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
9784 		ufshcd_disable_auto_bkops(hba);
9785 enable_scaling:
9786 	if (ufshcd_is_clkscaling_supported(hba))
9787 		ufshcd_clk_scaling_suspend(hba, false);
9788 
9789 	hba->dev_info.b_rpm_dev_flush_capable = false;
9790 out:
9791 	if (hba->dev_info.b_rpm_dev_flush_capable) {
9792 		schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
9793 			msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
9794 	}
9795 
9796 	if (ret) {
9797 		ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret);
9798 		hba->clk_gating.is_suspended = false;
9799 		ufshcd_release(hba);
9800 	}
9801 	hba->pm_op_in_progress = false;
9802 	return ret;
9803 }
9804 
9805 #ifdef CONFIG_PM
__ufshcd_wl_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)9806 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9807 {
9808 	int ret;
9809 	enum uic_link_state old_link_state = hba->uic_link_state;
9810 
9811 	hba->pm_op_in_progress = true;
9812 
9813 	/*
9814 	 * Call vendor specific resume callback. As these callbacks may access
9815 	 * vendor specific host controller register space call them when the
9816 	 * host clocks are ON.
9817 	 */
9818 	ret = ufshcd_vops_resume(hba, pm_op);
9819 	if (ret)
9820 		goto out;
9821 
9822 	/* For DeepSleep, the only supported option is to have the link off */
9823 	WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
9824 
9825 	if (ufshcd_is_link_hibern8(hba)) {
9826 		ret = ufshcd_uic_hibern8_exit(hba);
9827 		if (!ret) {
9828 			ufshcd_set_link_active(hba);
9829 		} else {
9830 			dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
9831 					__func__, ret);
9832 			goto vendor_suspend;
9833 		}
9834 	} else if (ufshcd_is_link_off(hba)) {
9835 		/*
9836 		 * A full initialization of the host and the device is
9837 		 * required since the link was put to off during suspend.
9838 		 * Note, in the case of DeepSleep, the device will exit
9839 		 * DeepSleep due to device reset.
9840 		 */
9841 		ret = ufshcd_reset_and_restore(hba);
9842 		/*
9843 		 * ufshcd_reset_and_restore() should have already
9844 		 * set the link state as active
9845 		 */
9846 		if (ret || !ufshcd_is_link_active(hba))
9847 			goto vendor_suspend;
9848 	}
9849 
9850 	if (!ufshcd_is_ufs_dev_active(hba)) {
9851 		ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
9852 		if (ret)
9853 			goto set_old_link_state;
9854 		ufshcd_set_timestamp_attr(hba);
9855 		schedule_delayed_work(&hba->ufs_rtc_update_work,
9856 				      msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS));
9857 	}
9858 
9859 	if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
9860 		ufshcd_enable_auto_bkops(hba);
9861 	else
9862 		/*
9863 		 * If BKOPs operations are urgently needed at this moment then
9864 		 * keep auto-bkops enabled or else disable it.
9865 		 */
9866 		ufshcd_bkops_ctrl(hba);
9867 
9868 	if (hba->ee_usr_mask)
9869 		ufshcd_write_ee_control(hba);
9870 
9871 	if (ufshcd_is_clkscaling_supported(hba))
9872 		ufshcd_clk_scaling_suspend(hba, false);
9873 
9874 	if (hba->dev_info.b_rpm_dev_flush_capable) {
9875 		hba->dev_info.b_rpm_dev_flush_capable = false;
9876 		cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
9877 	}
9878 
9879 	ufshcd_configure_auto_hibern8(hba);
9880 
9881 	goto out;
9882 
9883 set_old_link_state:
9884 	ufshcd_link_state_transition(hba, old_link_state, 0);
9885 vendor_suspend:
9886 	ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9887 	ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9888 out:
9889 	if (ret)
9890 		ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret);
9891 	hba->clk_gating.is_suspended = false;
9892 	ufshcd_release(hba);
9893 	hba->pm_op_in_progress = false;
9894 	return ret;
9895 }
9896 
ufshcd_wl_runtime_suspend(struct device * dev)9897 static int ufshcd_wl_runtime_suspend(struct device *dev)
9898 {
9899 	struct scsi_device *sdev = to_scsi_device(dev);
9900 	struct ufs_hba *hba;
9901 	int ret;
9902 	ktime_t start = ktime_get();
9903 
9904 	hba = shost_priv(sdev->host);
9905 
9906 	ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM);
9907 	if (ret)
9908 		dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9909 
9910 	trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret,
9911 		ktime_to_us(ktime_sub(ktime_get(), start)),
9912 		hba->curr_dev_pwr_mode, hba->uic_link_state);
9913 
9914 	return ret;
9915 }
9916 
ufshcd_wl_runtime_resume(struct device * dev)9917 static int ufshcd_wl_runtime_resume(struct device *dev)
9918 {
9919 	struct scsi_device *sdev = to_scsi_device(dev);
9920 	struct ufs_hba *hba;
9921 	int ret = 0;
9922 	ktime_t start = ktime_get();
9923 
9924 	hba = shost_priv(sdev->host);
9925 
9926 	ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM);
9927 	if (ret)
9928 		dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9929 
9930 	trace_ufshcd_wl_runtime_resume(dev_name(dev), ret,
9931 		ktime_to_us(ktime_sub(ktime_get(), start)),
9932 		hba->curr_dev_pwr_mode, hba->uic_link_state);
9933 
9934 	return ret;
9935 }
9936 #endif
9937 
9938 #ifdef CONFIG_PM_SLEEP
ufshcd_wl_suspend(struct device * dev)9939 static int ufshcd_wl_suspend(struct device *dev)
9940 {
9941 	struct scsi_device *sdev = to_scsi_device(dev);
9942 	struct ufs_hba *hba;
9943 	int ret = 0;
9944 	ktime_t start = ktime_get();
9945 
9946 	hba = shost_priv(sdev->host);
9947 	down(&hba->host_sem);
9948 	hba->system_suspending = true;
9949 
9950 	if (pm_runtime_suspended(dev))
9951 		goto out;
9952 
9953 	ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM);
9954 	if (ret) {
9955 		dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__,  ret);
9956 		up(&hba->host_sem);
9957 	}
9958 
9959 out:
9960 	if (!ret)
9961 		hba->is_sys_suspended = true;
9962 	trace_ufshcd_wl_suspend(dev_name(dev), ret,
9963 		ktime_to_us(ktime_sub(ktime_get(), start)),
9964 		hba->curr_dev_pwr_mode, hba->uic_link_state);
9965 
9966 	return ret;
9967 }
9968 
ufshcd_wl_resume(struct device * dev)9969 static int ufshcd_wl_resume(struct device *dev)
9970 {
9971 	struct scsi_device *sdev = to_scsi_device(dev);
9972 	struct ufs_hba *hba;
9973 	int ret = 0;
9974 	ktime_t start = ktime_get();
9975 
9976 	hba = shost_priv(sdev->host);
9977 
9978 	if (pm_runtime_suspended(dev))
9979 		goto out;
9980 
9981 	ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM);
9982 	if (ret)
9983 		dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9984 out:
9985 	trace_ufshcd_wl_resume(dev_name(dev), ret,
9986 		ktime_to_us(ktime_sub(ktime_get(), start)),
9987 		hba->curr_dev_pwr_mode, hba->uic_link_state);
9988 	if (!ret)
9989 		hba->is_sys_suspended = false;
9990 	hba->system_suspending = false;
9991 	up(&hba->host_sem);
9992 	return ret;
9993 }
9994 #endif
9995 
9996 /**
9997  * ufshcd_suspend - helper function for suspend operations
9998  * @hba: per adapter instance
9999  *
10000  * This function will put disable irqs, turn off clocks
10001  * and set vreg and hba-vreg in lpm mode.
10002  *
10003  * Return: 0 upon success; < 0 upon failure.
10004  */
ufshcd_suspend(struct ufs_hba * hba)10005 static int ufshcd_suspend(struct ufs_hba *hba)
10006 {
10007 	int ret;
10008 
10009 	if (!hba->is_powered)
10010 		return 0;
10011 	/*
10012 	 * Disable the host irq as host controller as there won't be any
10013 	 * host controller transaction expected till resume.
10014 	 */
10015 	ufshcd_disable_irq(hba);
10016 	ret = ufshcd_setup_clocks(hba, false);
10017 	if (ret) {
10018 		ufshcd_enable_irq(hba);
10019 		return ret;
10020 	}
10021 	if (ufshcd_is_clkgating_allowed(hba)) {
10022 		hba->clk_gating.state = CLKS_OFF;
10023 		trace_ufshcd_clk_gating(dev_name(hba->dev),
10024 					hba->clk_gating.state);
10025 	}
10026 
10027 	ufshcd_vreg_set_lpm(hba);
10028 	/* Put the host controller in low power mode if possible */
10029 	ufshcd_hba_vreg_set_lpm(hba);
10030 	ufshcd_pm_qos_update(hba, false);
10031 	return ret;
10032 }
10033 
10034 #ifdef CONFIG_PM
10035 /**
10036  * ufshcd_resume - helper function for resume operations
10037  * @hba: per adapter instance
10038  *
10039  * This function basically turns on the regulators, clocks and
10040  * irqs of the hba.
10041  *
10042  * Return: 0 for success and non-zero for failure.
10043  */
ufshcd_resume(struct ufs_hba * hba)10044 static int ufshcd_resume(struct ufs_hba *hba)
10045 {
10046 	int ret;
10047 
10048 	if (!hba->is_powered)
10049 		return 0;
10050 
10051 	ufshcd_hba_vreg_set_hpm(hba);
10052 	ret = ufshcd_vreg_set_hpm(hba);
10053 	if (ret)
10054 		goto out;
10055 
10056 	/* Make sure clocks are enabled before accessing controller */
10057 	ret = ufshcd_setup_clocks(hba, true);
10058 	if (ret)
10059 		goto disable_vreg;
10060 
10061 	/* enable the host irq as host controller would be active soon */
10062 	ufshcd_enable_irq(hba);
10063 
10064 	goto out;
10065 
10066 disable_vreg:
10067 	ufshcd_vreg_set_lpm(hba);
10068 out:
10069 	if (ret)
10070 		ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
10071 	return ret;
10072 }
10073 #endif /* CONFIG_PM */
10074 
10075 #ifdef CONFIG_PM_SLEEP
10076 /**
10077  * ufshcd_system_suspend - system suspend callback
10078  * @dev: Device associated with the UFS controller.
10079  *
10080  * Executed before putting the system into a sleep state in which the contents
10081  * of main memory are preserved.
10082  *
10083  * Return: 0 for success and non-zero for failure.
10084  */
ufshcd_system_suspend(struct device * dev)10085 int ufshcd_system_suspend(struct device *dev)
10086 {
10087 	struct ufs_hba *hba = dev_get_drvdata(dev);
10088 	int ret = 0;
10089 	ktime_t start = ktime_get();
10090 
10091 	if (pm_runtime_suspended(hba->dev))
10092 		goto out;
10093 
10094 	ret = ufshcd_suspend(hba);
10095 out:
10096 	trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
10097 		ktime_to_us(ktime_sub(ktime_get(), start)),
10098 		hba->curr_dev_pwr_mode, hba->uic_link_state);
10099 	return ret;
10100 }
10101 EXPORT_SYMBOL(ufshcd_system_suspend);
10102 
10103 /**
10104  * ufshcd_system_resume - system resume callback
10105  * @dev: Device associated with the UFS controller.
10106  *
10107  * Executed after waking the system up from a sleep state in which the contents
10108  * of main memory were preserved.
10109  *
10110  * Return: 0 for success and non-zero for failure.
10111  */
ufshcd_system_resume(struct device * dev)10112 int ufshcd_system_resume(struct device *dev)
10113 {
10114 	struct ufs_hba *hba = dev_get_drvdata(dev);
10115 	ktime_t start = ktime_get();
10116 	int ret = 0;
10117 
10118 	if (pm_runtime_suspended(hba->dev))
10119 		goto out;
10120 
10121 	ret = ufshcd_resume(hba);
10122 
10123 out:
10124 	trace_ufshcd_system_resume(dev_name(hba->dev), ret,
10125 		ktime_to_us(ktime_sub(ktime_get(), start)),
10126 		hba->curr_dev_pwr_mode, hba->uic_link_state);
10127 
10128 	return ret;
10129 }
10130 EXPORT_SYMBOL(ufshcd_system_resume);
10131 #endif /* CONFIG_PM_SLEEP */
10132 
10133 #ifdef CONFIG_PM
10134 /**
10135  * ufshcd_runtime_suspend - runtime suspend callback
10136  * @dev: Device associated with the UFS controller.
10137  *
10138  * Check the description of ufshcd_suspend() function for more details.
10139  *
10140  * Return: 0 for success and non-zero for failure.
10141  */
ufshcd_runtime_suspend(struct device * dev)10142 int ufshcd_runtime_suspend(struct device *dev)
10143 {
10144 	struct ufs_hba *hba = dev_get_drvdata(dev);
10145 	int ret;
10146 	ktime_t start = ktime_get();
10147 
10148 	ret = ufshcd_suspend(hba);
10149 
10150 	trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
10151 		ktime_to_us(ktime_sub(ktime_get(), start)),
10152 		hba->curr_dev_pwr_mode, hba->uic_link_state);
10153 	return ret;
10154 }
10155 EXPORT_SYMBOL(ufshcd_runtime_suspend);
10156 
10157 /**
10158  * ufshcd_runtime_resume - runtime resume routine
10159  * @dev: Device associated with the UFS controller.
10160  *
10161  * This function basically brings controller
10162  * to active state. Following operations are done in this function:
10163  *
10164  * 1. Turn on all the controller related clocks
10165  * 2. Turn ON VCC rail
10166  *
10167  * Return: 0 upon success; < 0 upon failure.
10168  */
ufshcd_runtime_resume(struct device * dev)10169 int ufshcd_runtime_resume(struct device *dev)
10170 {
10171 	struct ufs_hba *hba = dev_get_drvdata(dev);
10172 	int ret;
10173 	ktime_t start = ktime_get();
10174 
10175 	ret = ufshcd_resume(hba);
10176 
10177 	trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
10178 		ktime_to_us(ktime_sub(ktime_get(), start)),
10179 		hba->curr_dev_pwr_mode, hba->uic_link_state);
10180 	return ret;
10181 }
10182 EXPORT_SYMBOL(ufshcd_runtime_resume);
10183 #endif /* CONFIG_PM */
10184 
ufshcd_wl_shutdown(struct device * dev)10185 static void ufshcd_wl_shutdown(struct device *dev)
10186 {
10187 	struct scsi_device *sdev = to_scsi_device(dev);
10188 	struct ufs_hba *hba = shost_priv(sdev->host);
10189 
10190 	down(&hba->host_sem);
10191 	hba->shutting_down = true;
10192 	up(&hba->host_sem);
10193 
10194 	/* Turn on everything while shutting down */
10195 	ufshcd_rpm_get_sync(hba);
10196 	scsi_device_quiesce(sdev);
10197 	shost_for_each_device(sdev, hba->host) {
10198 		if (sdev == hba->ufs_device_wlun)
10199 			continue;
10200 		mutex_lock(&sdev->state_mutex);
10201 		scsi_device_set_state(sdev, SDEV_OFFLINE);
10202 		mutex_unlock(&sdev->state_mutex);
10203 	}
10204 	__ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
10205 
10206 	/*
10207 	 * Next, turn off the UFS controller and the UFS regulators. Disable
10208 	 * clocks.
10209 	 */
10210 	if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
10211 		ufshcd_suspend(hba);
10212 
10213 	hba->is_powered = false;
10214 }
10215 
10216 /**
10217  * ufshcd_remove - de-allocate SCSI host and host memory space
10218  *		data structure memory
10219  * @hba: per adapter instance
10220  */
ufshcd_remove(struct ufs_hba * hba)10221 void ufshcd_remove(struct ufs_hba *hba)
10222 {
10223 	if (hba->ufs_device_wlun)
10224 		ufshcd_rpm_get_sync(hba);
10225 	ufs_hwmon_remove(hba);
10226 	ufs_bsg_remove(hba);
10227 	ufs_sysfs_remove_nodes(hba->dev);
10228 	blk_mq_destroy_queue(hba->tmf_queue);
10229 	blk_put_queue(hba->tmf_queue);
10230 	blk_mq_free_tag_set(&hba->tmf_tag_set);
10231 	if (hba->scsi_host_added)
10232 		scsi_remove_host(hba->host);
10233 	/* disable interrupts */
10234 	ufshcd_disable_intr(hba, hba->intr_mask);
10235 	ufshcd_hba_stop(hba);
10236 	ufshcd_hba_exit(hba);
10237 }
10238 EXPORT_SYMBOL_GPL(ufshcd_remove);
10239 
10240 #ifdef CONFIG_PM_SLEEP
ufshcd_system_freeze(struct device * dev)10241 int ufshcd_system_freeze(struct device *dev)
10242 {
10243 
10244 	return ufshcd_system_suspend(dev);
10245 
10246 }
10247 EXPORT_SYMBOL_GPL(ufshcd_system_freeze);
10248 
ufshcd_system_restore(struct device * dev)10249 int ufshcd_system_restore(struct device *dev)
10250 {
10251 
10252 	struct ufs_hba *hba = dev_get_drvdata(dev);
10253 	int ret;
10254 
10255 	ret = ufshcd_system_resume(dev);
10256 	if (ret)
10257 		return ret;
10258 
10259 	/* Configure UTRL and UTMRL base address registers */
10260 	ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
10261 			REG_UTP_TRANSFER_REQ_LIST_BASE_L);
10262 	ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
10263 			REG_UTP_TRANSFER_REQ_LIST_BASE_H);
10264 	ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
10265 			REG_UTP_TASK_REQ_LIST_BASE_L);
10266 	ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
10267 			REG_UTP_TASK_REQ_LIST_BASE_H);
10268 	/*
10269 	 * Make sure that UTRL and UTMRL base address registers
10270 	 * are updated with the latest queue addresses. Only after
10271 	 * updating these addresses, we can queue the new commands.
10272 	 */
10273 	ufshcd_readl(hba, REG_UTP_TASK_REQ_LIST_BASE_H);
10274 
10275 	return 0;
10276 
10277 }
10278 EXPORT_SYMBOL_GPL(ufshcd_system_restore);
10279 
ufshcd_system_thaw(struct device * dev)10280 int ufshcd_system_thaw(struct device *dev)
10281 {
10282 	return ufshcd_system_resume(dev);
10283 }
10284 EXPORT_SYMBOL_GPL(ufshcd_system_thaw);
10285 #endif /* CONFIG_PM_SLEEP  */
10286 
10287 /**
10288  * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
10289  * @hba: pointer to Host Bus Adapter (HBA)
10290  */
ufshcd_dealloc_host(struct ufs_hba * hba)10291 void ufshcd_dealloc_host(struct ufs_hba *hba)
10292 {
10293 	scsi_host_put(hba->host);
10294 }
10295 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
10296 
10297 /**
10298  * ufshcd_set_dma_mask - Set dma mask based on the controller
10299  *			 addressing capability
10300  * @hba: per adapter instance
10301  *
10302  * Return: 0 for success, non-zero for failure.
10303  */
ufshcd_set_dma_mask(struct ufs_hba * hba)10304 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
10305 {
10306 	if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
10307 		if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
10308 			return 0;
10309 	}
10310 	return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
10311 }
10312 
10313 /**
10314  * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
10315  * @dev: pointer to device handle
10316  * @hba_handle: driver private handle
10317  *
10318  * Return: 0 on success, non-zero value on failure.
10319  */
ufshcd_alloc_host(struct device * dev,struct ufs_hba ** hba_handle)10320 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
10321 {
10322 	struct Scsi_Host *host;
10323 	struct ufs_hba *hba;
10324 	int err = 0;
10325 
10326 	if (!dev) {
10327 		dev_err(dev,
10328 		"Invalid memory reference for dev is NULL\n");
10329 		err = -ENODEV;
10330 		goto out_error;
10331 	}
10332 
10333 	host = scsi_host_alloc(&ufshcd_driver_template,
10334 				sizeof(struct ufs_hba));
10335 	if (!host) {
10336 		dev_err(dev, "scsi_host_alloc failed\n");
10337 		err = -ENOMEM;
10338 		goto out_error;
10339 	}
10340 	host->nr_maps = HCTX_TYPE_POLL + 1;
10341 	hba = shost_priv(host);
10342 	hba->host = host;
10343 	hba->dev = dev;
10344 	hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
10345 	hba->nop_out_timeout = NOP_OUT_TIMEOUT;
10346 	ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry));
10347 	INIT_LIST_HEAD(&hba->clk_list_head);
10348 	spin_lock_init(&hba->outstanding_lock);
10349 
10350 	*hba_handle = hba;
10351 
10352 out_error:
10353 	return err;
10354 }
10355 EXPORT_SYMBOL(ufshcd_alloc_host);
10356 
10357 /* This function exists because blk_mq_alloc_tag_set() requires this. */
ufshcd_queue_tmf(struct blk_mq_hw_ctx * hctx,const struct blk_mq_queue_data * qd)10358 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
10359 				     const struct blk_mq_queue_data *qd)
10360 {
10361 	WARN_ON_ONCE(true);
10362 	return BLK_STS_NOTSUPP;
10363 }
10364 
10365 static const struct blk_mq_ops ufshcd_tmf_ops = {
10366 	.queue_rq = ufshcd_queue_tmf,
10367 };
10368 
10369 /**
10370  * ufshcd_init - Driver initialization routine
10371  * @hba: per-adapter instance
10372  * @mmio_base: base register address
10373  * @irq: Interrupt line of device
10374  *
10375  * Return: 0 on success, non-zero value on failure.
10376  */
ufshcd_init(struct ufs_hba * hba,void __iomem * mmio_base,unsigned int irq)10377 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
10378 {
10379 	int err;
10380 	struct Scsi_Host *host = hba->host;
10381 	struct device *dev = hba->dev;
10382 
10383 	/*
10384 	 * dev_set_drvdata() must be called before any callbacks are registered
10385 	 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
10386 	 * sysfs).
10387 	 */
10388 	dev_set_drvdata(dev, hba);
10389 
10390 	if (!mmio_base) {
10391 		dev_err(hba->dev,
10392 		"Invalid memory reference for mmio_base is NULL\n");
10393 		err = -ENODEV;
10394 		goto out_error;
10395 	}
10396 
10397 	hba->mmio_base = mmio_base;
10398 	hba->irq = irq;
10399 	hba->vps = &ufs_hba_vps;
10400 
10401 	err = ufshcd_hba_init(hba);
10402 	if (err)
10403 		goto out_error;
10404 
10405 	/* Read capabilities registers */
10406 	err = ufshcd_hba_capabilities(hba);
10407 	if (err)
10408 		goto out_disable;
10409 
10410 	/* Get UFS version supported by the controller */
10411 	hba->ufs_version = ufshcd_get_ufs_version(hba);
10412 
10413 	/* Get Interrupt bit mask per version */
10414 	hba->intr_mask = ufshcd_get_intr_mask(hba);
10415 
10416 	err = ufshcd_set_dma_mask(hba);
10417 	if (err) {
10418 		dev_err(hba->dev, "set dma mask failed\n");
10419 		goto out_disable;
10420 	}
10421 
10422 	/* Allocate memory for host memory space */
10423 	err = ufshcd_memory_alloc(hba);
10424 	if (err) {
10425 		dev_err(hba->dev, "Memory allocation failed\n");
10426 		goto out_disable;
10427 	}
10428 
10429 	/* Configure LRB */
10430 	ufshcd_host_memory_configure(hba);
10431 
10432 	host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
10433 	host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
10434 	host->max_id = UFSHCD_MAX_ID;
10435 	host->max_lun = UFS_MAX_LUNS;
10436 	host->max_channel = UFSHCD_MAX_CHANNEL;
10437 	host->unique_id = host->host_no;
10438 	host->max_cmd_len = UFS_CDB_SIZE;
10439 	host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING);
10440 
10441 	/* Use default RPM delay if host not set */
10442 	if (host->rpm_autosuspend_delay == 0)
10443 		host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS;
10444 
10445 	hba->max_pwr_info.is_valid = false;
10446 
10447 	/* Initialize work queues */
10448 	hba->eh_wq = alloc_ordered_workqueue("ufs_eh_wq_%d", WQ_MEM_RECLAIM,
10449 					     hba->host->host_no);
10450 	if (!hba->eh_wq) {
10451 		dev_err(hba->dev, "%s: failed to create eh workqueue\n",
10452 			__func__);
10453 		err = -ENOMEM;
10454 		goto out_disable;
10455 	}
10456 	INIT_WORK(&hba->eh_work, ufshcd_err_handler);
10457 	INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
10458 
10459 	sema_init(&hba->host_sem, 1);
10460 
10461 	/* Initialize UIC command mutex */
10462 	mutex_init(&hba->uic_cmd_mutex);
10463 
10464 	/* Initialize mutex for device management commands */
10465 	mutex_init(&hba->dev_cmd.lock);
10466 
10467 	/* Initialize mutex for exception event control */
10468 	mutex_init(&hba->ee_ctrl_mutex);
10469 
10470 	mutex_init(&hba->wb_mutex);
10471 	init_rwsem(&hba->clk_scaling_lock);
10472 
10473 	ufshcd_init_clk_gating(hba);
10474 
10475 	ufshcd_init_clk_scaling(hba);
10476 
10477 	/*
10478 	 * In order to avoid any spurious interrupt immediately after
10479 	 * registering UFS controller interrupt handler, clear any pending UFS
10480 	 * interrupt status and disable all the UFS interrupts.
10481 	 */
10482 	ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
10483 		      REG_INTERRUPT_STATUS);
10484 	ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
10485 	/*
10486 	 * Make sure that UFS interrupts are disabled and any pending interrupt
10487 	 * status is cleared before registering UFS interrupt handler.
10488 	 */
10489 	ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
10490 
10491 	/* IRQ registration */
10492 	err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
10493 	if (err) {
10494 		dev_err(hba->dev, "request irq failed\n");
10495 		goto out_disable;
10496 	} else {
10497 		hba->is_irq_enabled = true;
10498 	}
10499 
10500 	if (!is_mcq_supported(hba)) {
10501 		if (!hba->lsdb_sup) {
10502 			dev_err(hba->dev, "%s: failed to initialize (legacy doorbell mode not supported)\n",
10503 				__func__);
10504 			err = -EINVAL;
10505 			goto out_disable;
10506 		}
10507 		err = scsi_add_host(host, hba->dev);
10508 		if (err) {
10509 			dev_err(hba->dev, "scsi_add_host failed\n");
10510 			goto out_disable;
10511 		}
10512 		hba->scsi_host_added = true;
10513 	}
10514 
10515 	hba->tmf_tag_set = (struct blk_mq_tag_set) {
10516 		.nr_hw_queues	= 1,
10517 		.queue_depth	= hba->nutmrs,
10518 		.ops		= &ufshcd_tmf_ops,
10519 		.flags		= BLK_MQ_F_NO_SCHED,
10520 	};
10521 	err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
10522 	if (err < 0)
10523 		goto out_remove_scsi_host;
10524 	hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL);
10525 	if (IS_ERR(hba->tmf_queue)) {
10526 		err = PTR_ERR(hba->tmf_queue);
10527 		goto free_tmf_tag_set;
10528 	}
10529 	hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs,
10530 				    sizeof(*hba->tmf_rqs), GFP_KERNEL);
10531 	if (!hba->tmf_rqs) {
10532 		err = -ENOMEM;
10533 		goto free_tmf_queue;
10534 	}
10535 
10536 	/* Reset the attached device */
10537 	ufshcd_device_reset(hba);
10538 
10539 	ufshcd_init_crypto(hba);
10540 
10541 	/* Host controller enable */
10542 	err = ufshcd_hba_enable(hba);
10543 	if (err) {
10544 		dev_err(hba->dev, "Host controller enable failed\n");
10545 		ufshcd_print_evt_hist(hba);
10546 		ufshcd_print_host_state(hba);
10547 		goto free_tmf_queue;
10548 	}
10549 
10550 	/*
10551 	 * Set the default power management level for runtime and system PM.
10552 	 * Default power saving mode is to keep UFS link in Hibern8 state
10553 	 * and UFS device in sleep state.
10554 	 */
10555 	hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10556 						UFS_SLEEP_PWR_MODE,
10557 						UIC_LINK_HIBERN8_STATE);
10558 	hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10559 						UFS_SLEEP_PWR_MODE,
10560 						UIC_LINK_HIBERN8_STATE);
10561 
10562 	INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work);
10563 	INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work);
10564 
10565 	/* Set the default auto-hiberate idle timer value to 150 ms */
10566 	if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
10567 		hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
10568 			    FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
10569 	}
10570 
10571 	/* Hold auto suspend until async scan completes */
10572 	pm_runtime_get_sync(dev);
10573 	atomic_set(&hba->scsi_block_reqs_cnt, 0);
10574 	/*
10575 	 * We are assuming that device wasn't put in sleep/power-down
10576 	 * state exclusively during the boot stage before kernel.
10577 	 * This assumption helps avoid doing link startup twice during
10578 	 * ufshcd_probe_hba().
10579 	 */
10580 	ufshcd_set_ufs_dev_active(hba);
10581 
10582 	async_schedule(ufshcd_async_scan, hba);
10583 	ufs_sysfs_add_nodes(hba->dev);
10584 
10585 	device_enable_async_suspend(dev);
10586 	ufshcd_pm_qos_init(hba);
10587 	return 0;
10588 
10589 free_tmf_queue:
10590 	blk_mq_destroy_queue(hba->tmf_queue);
10591 	blk_put_queue(hba->tmf_queue);
10592 free_tmf_tag_set:
10593 	blk_mq_free_tag_set(&hba->tmf_tag_set);
10594 out_remove_scsi_host:
10595 	if (hba->scsi_host_added)
10596 		scsi_remove_host(hba->host);
10597 out_disable:
10598 	hba->is_irq_enabled = false;
10599 	ufshcd_hba_exit(hba);
10600 out_error:
10601 	return err;
10602 }
10603 EXPORT_SYMBOL_GPL(ufshcd_init);
10604 
ufshcd_resume_complete(struct device * dev)10605 void ufshcd_resume_complete(struct device *dev)
10606 {
10607 	struct ufs_hba *hba = dev_get_drvdata(dev);
10608 
10609 	if (hba->complete_put) {
10610 		ufshcd_rpm_put(hba);
10611 		hba->complete_put = false;
10612 	}
10613 }
10614 EXPORT_SYMBOL_GPL(ufshcd_resume_complete);
10615 
ufshcd_rpm_ok_for_spm(struct ufs_hba * hba)10616 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba)
10617 {
10618 	struct device *dev = &hba->ufs_device_wlun->sdev_gendev;
10619 	enum ufs_dev_pwr_mode dev_pwr_mode;
10620 	enum uic_link_state link_state;
10621 	unsigned long flags;
10622 	bool res;
10623 
10624 	spin_lock_irqsave(&dev->power.lock, flags);
10625 	dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl);
10626 	link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl);
10627 	res = pm_runtime_suspended(dev) &&
10628 	      hba->curr_dev_pwr_mode == dev_pwr_mode &&
10629 	      hba->uic_link_state == link_state &&
10630 	      !hba->dev_info.b_rpm_dev_flush_capable;
10631 	spin_unlock_irqrestore(&dev->power.lock, flags);
10632 
10633 	return res;
10634 }
10635 
__ufshcd_suspend_prepare(struct device * dev,bool rpm_ok_for_spm)10636 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm)
10637 {
10638 	struct ufs_hba *hba = dev_get_drvdata(dev);
10639 	int ret;
10640 
10641 	/*
10642 	 * SCSI assumes that runtime-pm and system-pm for scsi drivers
10643 	 * are same. And it doesn't wake up the device for system-suspend
10644 	 * if it's runtime suspended. But ufs doesn't follow that.
10645 	 * Refer ufshcd_resume_complete()
10646 	 */
10647 	if (hba->ufs_device_wlun) {
10648 		/* Prevent runtime suspend */
10649 		ufshcd_rpm_get_noresume(hba);
10650 		/*
10651 		 * Check if already runtime suspended in same state as system
10652 		 * suspend would be.
10653 		 */
10654 		if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) {
10655 			/* RPM state is not ok for SPM, so runtime resume */
10656 			ret = ufshcd_rpm_resume(hba);
10657 			if (ret < 0 && ret != -EACCES) {
10658 				ufshcd_rpm_put(hba);
10659 				return ret;
10660 			}
10661 		}
10662 		hba->complete_put = true;
10663 	}
10664 	return 0;
10665 }
10666 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare);
10667 
ufshcd_suspend_prepare(struct device * dev)10668 int ufshcd_suspend_prepare(struct device *dev)
10669 {
10670 	return __ufshcd_suspend_prepare(dev, true);
10671 }
10672 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare);
10673 
10674 #ifdef CONFIG_PM_SLEEP
ufshcd_wl_poweroff(struct device * dev)10675 static int ufshcd_wl_poweroff(struct device *dev)
10676 {
10677 	struct scsi_device *sdev = to_scsi_device(dev);
10678 	struct ufs_hba *hba = shost_priv(sdev->host);
10679 
10680 	__ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
10681 	return 0;
10682 }
10683 #endif
10684 
ufshcd_wl_probe(struct device * dev)10685 static int ufshcd_wl_probe(struct device *dev)
10686 {
10687 	struct scsi_device *sdev = to_scsi_device(dev);
10688 
10689 	if (!is_device_wlun(sdev))
10690 		return -ENODEV;
10691 
10692 	blk_pm_runtime_init(sdev->request_queue, dev);
10693 	pm_runtime_set_autosuspend_delay(dev, 0);
10694 	pm_runtime_allow(dev);
10695 
10696 	return  0;
10697 }
10698 
ufshcd_wl_remove(struct device * dev)10699 static int ufshcd_wl_remove(struct device *dev)
10700 {
10701 	pm_runtime_forbid(dev);
10702 	return 0;
10703 }
10704 
10705 static const struct dev_pm_ops ufshcd_wl_pm_ops = {
10706 #ifdef CONFIG_PM_SLEEP
10707 	.suspend = ufshcd_wl_suspend,
10708 	.resume = ufshcd_wl_resume,
10709 	.freeze = ufshcd_wl_suspend,
10710 	.thaw = ufshcd_wl_resume,
10711 	.poweroff = ufshcd_wl_poweroff,
10712 	.restore = ufshcd_wl_resume,
10713 #endif
10714 	SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL)
10715 };
10716 
ufshcd_check_header_layout(void)10717 static void ufshcd_check_header_layout(void)
10718 {
10719 	/*
10720 	 * gcc compilers before version 10 cannot do constant-folding for
10721 	 * sub-byte bitfields. Hence skip the layout checks for gcc 9 and
10722 	 * before.
10723 	 */
10724 	if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000)
10725 		return;
10726 
10727 	BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10728 				.cci = 3})[0] != 3);
10729 
10730 	BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10731 				.ehs_length = 2})[1] != 2);
10732 
10733 	BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10734 				.enable_crypto = 1})[2]
10735 		     != 0x80);
10736 
10737 	BUILD_BUG_ON((((u8 *)&(struct request_desc_header){
10738 					.command_type = 5,
10739 					.data_direction = 3,
10740 					.interrupt = 1,
10741 				})[3]) != ((5 << 4) | (3 << 1) | 1));
10742 
10743 	BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){
10744 				.dunl = cpu_to_le32(0xdeadbeef)})[1] !=
10745 		cpu_to_le32(0xdeadbeef));
10746 
10747 	BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10748 				.ocs = 4})[8] != 4);
10749 
10750 	BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10751 				.cds = 5})[9] != 5);
10752 
10753 	BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){
10754 				.dunu = cpu_to_le32(0xbadcafe)})[3] !=
10755 		cpu_to_le32(0xbadcafe));
10756 
10757 	BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){
10758 			     .iid = 0xf })[4] != 0xf0);
10759 
10760 	BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){
10761 			     .command_set_type = 0xf })[4] != 0xf);
10762 }
10763 
10764 /*
10765  * ufs_dev_wlun_template - describes ufs device wlun
10766  * ufs-device wlun - used to send pm commands
10767  * All luns are consumers of ufs-device wlun.
10768  *
10769  * Currently, no sd driver is present for wluns.
10770  * Hence the no specific pm operations are performed.
10771  * With ufs design, SSU should be sent to ufs-device wlun.
10772  * Hence register a scsi driver for ufs wluns only.
10773  */
10774 static struct scsi_driver ufs_dev_wlun_template = {
10775 	.gendrv = {
10776 		.name = "ufs_device_wlun",
10777 		.probe = ufshcd_wl_probe,
10778 		.remove = ufshcd_wl_remove,
10779 		.pm = &ufshcd_wl_pm_ops,
10780 		.shutdown = ufshcd_wl_shutdown,
10781 	},
10782 };
10783 
ufshcd_core_init(void)10784 static int __init ufshcd_core_init(void)
10785 {
10786 	int ret;
10787 
10788 	ufshcd_check_header_layout();
10789 
10790 	ufs_debugfs_init();
10791 
10792 	ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
10793 	if (ret)
10794 		ufs_debugfs_exit();
10795 	return ret;
10796 }
10797 
ufshcd_core_exit(void)10798 static void __exit ufshcd_core_exit(void)
10799 {
10800 	ufs_debugfs_exit();
10801 	scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
10802 }
10803 
10804 module_init(ufshcd_core_init);
10805 module_exit(ufshcd_core_exit);
10806 
10807 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
10808 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
10809 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
10810 MODULE_SOFTDEP("pre: governor_simpleondemand");
10811 MODULE_LICENSE("GPL");
10812