1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Universal Flash Storage Host controller driver Core
4 * Copyright (C) 2011-2013 Samsung India Software Operations
5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
6 *
7 * Authors:
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
10 */
11
12 #include <linux/async.h>
13 #include <linux/devfreq.h>
14 #include <linux/nls.h>
15 #include <linux/of.h>
16 #include <linux/bitfield.h>
17 #include <linux/blk-pm.h>
18 #include <linux/blkdev.h>
19 #include <linux/clk.h>
20 #include <linux/delay.h>
21 #include <linux/interrupt.h>
22 #include <linux/module.h>
23 #include <linux/pm_opp.h>
24 #include <linux/regulator/consumer.h>
25 #include <linux/sched/clock.h>
26 #include <linux/iopoll.h>
27 #include <scsi/scsi_cmnd.h>
28 #include <scsi/scsi_dbg.h>
29 #include <scsi/scsi_driver.h>
30 #include <scsi/scsi_eh.h>
31 #include "ufshcd-priv.h"
32 #include <ufs/ufs_quirks.h>
33 #include <ufs/unipro.h>
34 #include "ufs-sysfs.h"
35 #include "ufs-debugfs.h"
36 #include "ufs-fault-injection.h"
37 #include "ufs_bsg.h"
38 #include "ufshcd-crypto.h"
39 #include <linux/unaligned.h>
40
41 #define CREATE_TRACE_POINTS
42 #include "ufs_trace.h"
43
44 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
45 UTP_TASK_REQ_COMPL |\
46 UFSHCD_ERROR_MASK)
47
48 #define UFSHCD_ENABLE_MCQ_INTRS (UTP_TASK_REQ_COMPL |\
49 UFSHCD_ERROR_MASK |\
50 MCQ_CQ_EVENT_STATUS)
51
52
53 /* UIC command timeout, unit: ms */
54 enum {
55 UIC_CMD_TIMEOUT_DEFAULT = 500,
56 UIC_CMD_TIMEOUT_MAX = 2000,
57 };
58 /* NOP OUT retries waiting for NOP IN response */
59 #define NOP_OUT_RETRIES 10
60 /* Timeout after 50 msecs if NOP OUT hangs without response */
61 #define NOP_OUT_TIMEOUT 50 /* msecs */
62
63 /* Query request retries */
64 #define QUERY_REQ_RETRIES 3
65 /* Query request timeout */
66 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
67
68 /* Advanced RPMB request timeout */
69 #define ADVANCED_RPMB_REQ_TIMEOUT 3000 /* 3 seconds */
70
71 /* Task management command timeout */
72 #define TM_CMD_TIMEOUT 100 /* msecs */
73
74 /* maximum number of retries for a general UIC command */
75 #define UFS_UIC_COMMAND_RETRIES 3
76
77 /* maximum number of link-startup retries */
78 #define DME_LINKSTARTUP_RETRIES 3
79
80 /* maximum number of reset retries before giving up */
81 #define MAX_HOST_RESET_RETRIES 5
82
83 /* Maximum number of error handler retries before giving up */
84 #define MAX_ERR_HANDLER_RETRIES 5
85
86 /* Expose the flag value from utp_upiu_query.value */
87 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
88
89 /* Interrupt aggregation default timeout, unit: 40us */
90 #define INT_AGGR_DEF_TO 0x02
91
92 /* default delay of autosuspend: 2000 ms */
93 #define RPM_AUTOSUSPEND_DELAY_MS 2000
94
95 /* Default delay of RPM device flush delayed work */
96 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
97
98 /* Default value of wait time before gating device ref clock */
99 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
100
101 /* Polling time to wait for fDeviceInit */
102 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
103
104 /* Default RTC update every 10 seconds */
105 #define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC)
106
107 /* bMaxNumOfRTT is equal to two after device manufacturing */
108 #define DEFAULT_MAX_NUM_RTT 2
109
110 /* UFSHC 4.0 compliant HC support this mode. */
111 static bool use_mcq_mode = true;
112
is_mcq_supported(struct ufs_hba * hba)113 static bool is_mcq_supported(struct ufs_hba *hba)
114 {
115 return hba->mcq_sup && use_mcq_mode;
116 }
117
118 module_param(use_mcq_mode, bool, 0644);
119 MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default");
120
121 static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT;
122
uic_cmd_timeout_set(const char * val,const struct kernel_param * kp)123 static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp)
124 {
125 return param_set_uint_minmax(val, kp, UIC_CMD_TIMEOUT_DEFAULT,
126 UIC_CMD_TIMEOUT_MAX);
127 }
128
129 static const struct kernel_param_ops uic_cmd_timeout_ops = {
130 .set = uic_cmd_timeout_set,
131 .get = param_get_uint,
132 };
133
134 module_param_cb(uic_cmd_timeout, &uic_cmd_timeout_ops, &uic_cmd_timeout, 0644);
135 MODULE_PARM_DESC(uic_cmd_timeout,
136 "UFS UIC command timeout in milliseconds. Defaults to 500ms. Supported values range from 500ms to 2 seconds inclusively");
137
138 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \
139 ({ \
140 int _ret; \
141 if (_on) \
142 _ret = ufshcd_enable_vreg(_dev, _vreg); \
143 else \
144 _ret = ufshcd_disable_vreg(_dev, _vreg); \
145 _ret; \
146 })
147
148 #define ufshcd_hex_dump(prefix_str, buf, len) do { \
149 size_t __len = (len); \
150 print_hex_dump(KERN_ERR, prefix_str, \
151 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
152 16, 4, buf, __len, false); \
153 } while (0)
154
ufshcd_dump_regs(struct ufs_hba * hba,size_t offset,size_t len,const char * prefix)155 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
156 const char *prefix)
157 {
158 u32 *regs;
159 size_t pos;
160
161 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
162 return -EINVAL;
163
164 regs = kzalloc(len, GFP_ATOMIC);
165 if (!regs)
166 return -ENOMEM;
167
168 for (pos = 0; pos < len; pos += 4) {
169 if (offset == 0 &&
170 pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER &&
171 pos <= REG_UIC_ERROR_CODE_DME)
172 continue;
173 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
174 }
175
176 ufshcd_hex_dump(prefix, regs, len);
177 kfree(regs);
178
179 return 0;
180 }
181 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
182
183 enum {
184 UFSHCD_MAX_CHANNEL = 0,
185 UFSHCD_MAX_ID = 1,
186 };
187
188 static const char *const ufshcd_state_name[] = {
189 [UFSHCD_STATE_RESET] = "reset",
190 [UFSHCD_STATE_OPERATIONAL] = "operational",
191 [UFSHCD_STATE_ERROR] = "error",
192 [UFSHCD_STATE_EH_SCHEDULED_FATAL] = "eh_fatal",
193 [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL] = "eh_non_fatal",
194 };
195
196 /* UFSHCD error handling flags */
197 enum {
198 UFSHCD_EH_IN_PROGRESS = (1 << 0),
199 };
200
201 /* UFSHCD UIC layer error flags */
202 enum {
203 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
204 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
205 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
206 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
207 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
208 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
209 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
210 };
211
212 #define ufshcd_set_eh_in_progress(h) \
213 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
214 #define ufshcd_eh_in_progress(h) \
215 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
216 #define ufshcd_clear_eh_in_progress(h) \
217 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
218
219 const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
220 [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
221 [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
222 [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
223 [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
224 [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
225 [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
226 /*
227 * For DeepSleep, the link is first put in hibern8 and then off.
228 * Leaving the link in hibern8 is not supported.
229 */
230 [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
231 };
232
233 static inline enum ufs_dev_pwr_mode
ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)234 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
235 {
236 return ufs_pm_lvl_states[lvl].dev_state;
237 }
238
239 static inline enum uic_link_state
ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)240 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
241 {
242 return ufs_pm_lvl_states[lvl].link_state;
243 }
244
245 static inline enum ufs_pm_level
ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,enum uic_link_state link_state)246 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
247 enum uic_link_state link_state)
248 {
249 enum ufs_pm_level lvl;
250
251 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
252 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
253 (ufs_pm_lvl_states[lvl].link_state == link_state))
254 return lvl;
255 }
256
257 /* if no match found, return the level 0 */
258 return UFS_PM_LVL_0;
259 }
260
ufshcd_is_ufs_dev_busy(struct ufs_hba * hba)261 static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba)
262 {
263 return (hba->clk_gating.active_reqs || hba->outstanding_reqs || hba->outstanding_tasks ||
264 hba->active_uic_cmd || hba->uic_async_done);
265 }
266
267 static const struct ufs_dev_quirk ufs_fixups[] = {
268 /* UFS cards deviations table */
269 { .wmanufacturerid = UFS_VENDOR_MICRON,
270 .model = UFS_ANY_MODEL,
271 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
272 { .wmanufacturerid = UFS_VENDOR_SAMSUNG,
273 .model = UFS_ANY_MODEL,
274 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
275 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
276 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS },
277 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
278 .model = UFS_ANY_MODEL,
279 .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME },
280 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
281 .model = "hB8aL1" /*H28U62301AMR*/,
282 .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME },
283 { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
284 .model = UFS_ANY_MODEL,
285 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
286 { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
287 .model = "THGLF2G9C8KBADG",
288 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
289 { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
290 .model = "THGLF2G9D8KBADG",
291 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
292 {}
293 };
294
295 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
296 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
297 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
298 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
299 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
300 static void ufshcd_hba_exit(struct ufs_hba *hba);
301 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params);
302 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
303 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
304 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
305 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
306 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
307 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq,
308 bool scale_up);
309 static irqreturn_t ufshcd_intr(int irq, void *__hba);
310 static int ufshcd_change_power_mode(struct ufs_hba *hba,
311 struct ufs_pa_layer_attr *pwr_mode);
312 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
313 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
314 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
315 struct ufs_vreg *vreg);
316 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
317 bool enable);
318 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
319 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
320
ufshcd_enable_irq(struct ufs_hba * hba)321 void ufshcd_enable_irq(struct ufs_hba *hba)
322 {
323 if (!hba->is_irq_enabled) {
324 enable_irq(hba->irq);
325 hba->is_irq_enabled = true;
326 }
327 }
328 EXPORT_SYMBOL_GPL(ufshcd_enable_irq);
329
ufshcd_disable_irq(struct ufs_hba * hba)330 void ufshcd_disable_irq(struct ufs_hba *hba)
331 {
332 if (hba->is_irq_enabled) {
333 disable_irq(hba->irq);
334 hba->is_irq_enabled = false;
335 }
336 }
337 EXPORT_SYMBOL_GPL(ufshcd_disable_irq);
338
ufshcd_configure_wb(struct ufs_hba * hba)339 static void ufshcd_configure_wb(struct ufs_hba *hba)
340 {
341 if (!ufshcd_is_wb_allowed(hba))
342 return;
343
344 ufshcd_wb_toggle(hba, true);
345
346 ufshcd_wb_toggle_buf_flush_during_h8(hba, true);
347
348 if (ufshcd_is_wb_buf_flush_allowed(hba))
349 ufshcd_wb_toggle_buf_flush(hba, true);
350 }
351
ufshcd_scsi_unblock_requests(struct ufs_hba * hba)352 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
353 {
354 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
355 scsi_unblock_requests(hba->host);
356 }
357
ufshcd_scsi_block_requests(struct ufs_hba * hba)358 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
359 {
360 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
361 scsi_block_requests(hba->host);
362 }
363
ufshcd_add_cmd_upiu_trace(struct ufs_hba * hba,unsigned int tag,enum ufs_trace_str_t str_t)364 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
365 enum ufs_trace_str_t str_t)
366 {
367 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
368 struct utp_upiu_header *header;
369
370 if (!trace_ufshcd_upiu_enabled())
371 return;
372
373 if (str_t == UFS_CMD_SEND)
374 header = &rq->header;
375 else
376 header = &hba->lrb[tag].ucd_rsp_ptr->header;
377
378 trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
379 UFS_TSF_CDB);
380 }
381
ufshcd_add_query_upiu_trace(struct ufs_hba * hba,enum ufs_trace_str_t str_t,struct utp_upiu_req * rq_rsp)382 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
383 enum ufs_trace_str_t str_t,
384 struct utp_upiu_req *rq_rsp)
385 {
386 if (!trace_ufshcd_upiu_enabled())
387 return;
388
389 trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
390 &rq_rsp->qr, UFS_TSF_OSF);
391 }
392
ufshcd_add_tm_upiu_trace(struct ufs_hba * hba,unsigned int tag,enum ufs_trace_str_t str_t)393 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
394 enum ufs_trace_str_t str_t)
395 {
396 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
397
398 if (!trace_ufshcd_upiu_enabled())
399 return;
400
401 if (str_t == UFS_TM_SEND)
402 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
403 &descp->upiu_req.req_header,
404 &descp->upiu_req.input_param1,
405 UFS_TSF_TM_INPUT);
406 else
407 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
408 &descp->upiu_rsp.rsp_header,
409 &descp->upiu_rsp.output_param1,
410 UFS_TSF_TM_OUTPUT);
411 }
412
ufshcd_add_uic_command_trace(struct ufs_hba * hba,const struct uic_command * ucmd,enum ufs_trace_str_t str_t)413 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
414 const struct uic_command *ucmd,
415 enum ufs_trace_str_t str_t)
416 {
417 u32 cmd;
418
419 if (!trace_ufshcd_uic_command_enabled())
420 return;
421
422 if (str_t == UFS_CMD_SEND)
423 cmd = ucmd->command;
424 else
425 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
426
427 trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
428 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
429 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
430 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
431 }
432
ufshcd_add_command_trace(struct ufs_hba * hba,unsigned int tag,enum ufs_trace_str_t str_t)433 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
434 enum ufs_trace_str_t str_t)
435 {
436 u64 lba = 0;
437 u8 opcode = 0, group_id = 0;
438 u32 doorbell = 0;
439 u32 intr;
440 int hwq_id = -1;
441 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
442 struct scsi_cmnd *cmd = lrbp->cmd;
443 struct request *rq = scsi_cmd_to_rq(cmd);
444 int transfer_len = -1;
445
446 if (!cmd)
447 return;
448
449 /* trace UPIU also */
450 ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
451 if (!trace_ufshcd_command_enabled())
452 return;
453
454 opcode = cmd->cmnd[0];
455
456 if (opcode == READ_10 || opcode == WRITE_10) {
457 /*
458 * Currently we only fully trace read(10) and write(10) commands
459 */
460 transfer_len =
461 be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
462 lba = scsi_get_lba(cmd);
463 if (opcode == WRITE_10)
464 group_id = lrbp->cmd->cmnd[6];
465 } else if (opcode == UNMAP) {
466 /*
467 * The number of Bytes to be unmapped beginning with the lba.
468 */
469 transfer_len = blk_rq_bytes(rq);
470 lba = scsi_get_lba(cmd);
471 }
472
473 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
474
475 if (hba->mcq_enabled) {
476 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
477
478 hwq_id = hwq->id;
479 } else {
480 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
481 }
482 trace_ufshcd_command(cmd->device, str_t, tag, doorbell, hwq_id,
483 transfer_len, intr, lba, opcode, group_id);
484 }
485
ufshcd_print_clk_freqs(struct ufs_hba * hba)486 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
487 {
488 struct ufs_clk_info *clki;
489 struct list_head *head = &hba->clk_list_head;
490
491 if (list_empty(head))
492 return;
493
494 list_for_each_entry(clki, head, list) {
495 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
496 clki->max_freq)
497 dev_err(hba->dev, "clk: %s, rate: %u\n",
498 clki->name, clki->curr_freq);
499 }
500 }
501
ufshcd_print_evt(struct ufs_hba * hba,u32 id,const char * err_name)502 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
503 const char *err_name)
504 {
505 int i;
506 bool found = false;
507 const struct ufs_event_hist *e;
508
509 if (id >= UFS_EVT_CNT)
510 return;
511
512 e = &hba->ufs_stats.event[id];
513
514 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
515 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
516
517 if (e->tstamp[p] == 0)
518 continue;
519 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
520 e->val[p], div_u64(e->tstamp[p], 1000));
521 found = true;
522 }
523
524 if (!found)
525 dev_err(hba->dev, "No record of %s\n", err_name);
526 else
527 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
528 }
529
ufshcd_print_evt_hist(struct ufs_hba * hba)530 static void ufshcd_print_evt_hist(struct ufs_hba *hba)
531 {
532 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
533
534 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
535 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
536 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
537 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
538 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
539 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
540 "auto_hibern8_err");
541 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
542 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
543 "link_startup_fail");
544 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
545 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
546 "suspend_fail");
547 ufshcd_print_evt(hba, UFS_EVT_WL_RES_ERR, "wlun resume_fail");
548 ufshcd_print_evt(hba, UFS_EVT_WL_SUSP_ERR,
549 "wlun suspend_fail");
550 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
551 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
552 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
553
554 ufshcd_vops_dbg_register_dump(hba);
555 }
556
557 static
ufshcd_print_tr(struct ufs_hba * hba,int tag,bool pr_prdt)558 void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
559 {
560 const struct ufshcd_lrb *lrbp;
561 int prdt_length;
562
563 lrbp = &hba->lrb[tag];
564
565 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
566 tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
567 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
568 tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000));
569 dev_err(hba->dev,
570 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
571 tag, (u64)lrbp->utrd_dma_addr);
572
573 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
574 sizeof(struct utp_transfer_req_desc));
575 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
576 (u64)lrbp->ucd_req_dma_addr);
577 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
578 sizeof(struct utp_upiu_req));
579 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
580 (u64)lrbp->ucd_rsp_dma_addr);
581 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
582 sizeof(struct utp_upiu_rsp));
583
584 prdt_length = le16_to_cpu(
585 lrbp->utr_descriptor_ptr->prd_table_length);
586 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
587 prdt_length /= ufshcd_sg_entry_size(hba);
588
589 dev_err(hba->dev,
590 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
591 tag, prdt_length,
592 (u64)lrbp->ucd_prdt_dma_addr);
593
594 if (pr_prdt)
595 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
596 ufshcd_sg_entry_size(hba) * prdt_length);
597 }
598
ufshcd_print_tr_iter(struct request * req,void * priv)599 static bool ufshcd_print_tr_iter(struct request *req, void *priv)
600 {
601 struct scsi_device *sdev = req->q->queuedata;
602 struct Scsi_Host *shost = sdev->host;
603 struct ufs_hba *hba = shost_priv(shost);
604
605 ufshcd_print_tr(hba, req->tag, *(bool *)priv);
606
607 return true;
608 }
609
610 /**
611 * ufshcd_print_trs_all - print trs for all started requests.
612 * @hba: per-adapter instance.
613 * @pr_prdt: need to print prdt or not.
614 */
ufshcd_print_trs_all(struct ufs_hba * hba,bool pr_prdt)615 static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt)
616 {
617 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt);
618 }
619
ufshcd_print_tmrs(struct ufs_hba * hba,unsigned long bitmap)620 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
621 {
622 int tag;
623
624 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
625 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
626
627 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
628 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
629 }
630 }
631
ufshcd_print_host_state(struct ufs_hba * hba)632 static void ufshcd_print_host_state(struct ufs_hba *hba)
633 {
634 const struct scsi_device *sdev_ufs = hba->ufs_device_wlun;
635
636 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
637 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
638 hba->outstanding_reqs, hba->outstanding_tasks);
639 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
640 hba->saved_err, hba->saved_uic_err);
641 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
642 hba->curr_dev_pwr_mode, hba->uic_link_state);
643 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
644 hba->pm_op_in_progress, hba->is_sys_suspended);
645 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
646 hba->auto_bkops_enabled, hba->host->host_self_blocked);
647 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
648 dev_err(hba->dev,
649 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
650 div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000),
651 hba->ufs_stats.hibern8_exit_cnt);
652 dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
653 div_u64(hba->ufs_stats.last_intr_ts, 1000),
654 hba->ufs_stats.last_intr_status);
655 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
656 hba->eh_flags, hba->req_abort_count);
657 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
658 hba->ufs_version, hba->capabilities, hba->caps);
659 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
660 hba->dev_quirks);
661 if (sdev_ufs)
662 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
663 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
664
665 ufshcd_print_clk_freqs(hba);
666 }
667
668 /**
669 * ufshcd_print_pwr_info - print power params as saved in hba
670 * power info
671 * @hba: per-adapter instance
672 */
ufshcd_print_pwr_info(struct ufs_hba * hba)673 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
674 {
675 static const char * const names[] = {
676 "INVALID MODE",
677 "FAST MODE",
678 "SLOW_MODE",
679 "INVALID MODE",
680 "FASTAUTO_MODE",
681 "SLOWAUTO_MODE",
682 "INVALID MODE",
683 };
684
685 /*
686 * Using dev_dbg to avoid messages during runtime PM to avoid
687 * never-ending cycles of messages written back to storage by user space
688 * causing runtime resume, causing more messages and so on.
689 */
690 dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
691 __func__,
692 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
693 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
694 names[hba->pwr_info.pwr_rx],
695 names[hba->pwr_info.pwr_tx],
696 hba->pwr_info.hs_rate);
697 }
698
ufshcd_device_reset(struct ufs_hba * hba)699 static void ufshcd_device_reset(struct ufs_hba *hba)
700 {
701 int err;
702
703 err = ufshcd_vops_device_reset(hba);
704
705 if (!err) {
706 ufshcd_set_ufs_dev_active(hba);
707 if (ufshcd_is_wb_allowed(hba)) {
708 hba->dev_info.wb_enabled = false;
709 hba->dev_info.wb_buf_flush_enabled = false;
710 }
711 if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE)
712 hba->dev_info.rtc_time_baseline = 0;
713 }
714 if (err != -EOPNOTSUPP)
715 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
716 }
717
ufshcd_delay_us(unsigned long us,unsigned long tolerance)718 void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
719 {
720 if (!us)
721 return;
722
723 if (us < 10)
724 udelay(us);
725 else
726 usleep_range(us, us + tolerance);
727 }
728 EXPORT_SYMBOL_GPL(ufshcd_delay_us);
729
730 /**
731 * ufshcd_wait_for_register - wait for register value to change
732 * @hba: per-adapter interface
733 * @reg: mmio register offset
734 * @mask: mask to apply to the read register value
735 * @val: value to wait for
736 * @interval_us: polling interval in microseconds
737 * @timeout_ms: timeout in milliseconds
738 *
739 * Return: -ETIMEDOUT on error, zero on success.
740 */
ufshcd_wait_for_register(struct ufs_hba * hba,u32 reg,u32 mask,u32 val,unsigned long interval_us,unsigned long timeout_ms)741 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
742 u32 val, unsigned long interval_us,
743 unsigned long timeout_ms)
744 {
745 int err = 0;
746 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
747
748 /* ignore bits that we don't intend to wait on */
749 val = val & mask;
750
751 while ((ufshcd_readl(hba, reg) & mask) != val) {
752 usleep_range(interval_us, interval_us + 50);
753 if (time_after(jiffies, timeout)) {
754 if ((ufshcd_readl(hba, reg) & mask) != val)
755 err = -ETIMEDOUT;
756 break;
757 }
758 }
759
760 return err;
761 }
762
763 /**
764 * ufshcd_get_intr_mask - Get the interrupt bit mask
765 * @hba: Pointer to adapter instance
766 *
767 * Return: interrupt bit mask per version
768 */
ufshcd_get_intr_mask(struct ufs_hba * hba)769 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
770 {
771 if (hba->ufs_version <= ufshci_version(2, 0))
772 return INTERRUPT_MASK_ALL_VER_11;
773
774 return INTERRUPT_MASK_ALL_VER_21;
775 }
776
777 /**
778 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
779 * @hba: Pointer to adapter instance
780 *
781 * Return: UFSHCI version supported by the controller
782 */
ufshcd_get_ufs_version(struct ufs_hba * hba)783 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
784 {
785 u32 ufshci_ver;
786
787 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
788 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
789 else
790 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
791
792 /*
793 * UFSHCI v1.x uses a different version scheme, in order
794 * to allow the use of comparisons with the ufshci_version
795 * function, we convert it to the same scheme as ufs 2.0+.
796 */
797 if (ufshci_ver & 0x00010000)
798 return ufshci_version(1, ufshci_ver & 0x00000100);
799
800 return ufshci_ver;
801 }
802
803 /**
804 * ufshcd_is_device_present - Check if any device connected to
805 * the host controller
806 * @hba: pointer to adapter instance
807 *
808 * Return: true if device present, false if no device detected
809 */
ufshcd_is_device_present(struct ufs_hba * hba)810 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
811 {
812 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT;
813 }
814
815 /**
816 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
817 * @lrbp: pointer to local command reference block
818 * @cqe: pointer to the completion queue entry
819 *
820 * This function is used to get the OCS field from UTRD
821 *
822 * Return: the OCS field in the UTRD.
823 */
ufshcd_get_tr_ocs(struct ufshcd_lrb * lrbp,struct cq_entry * cqe)824 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp,
825 struct cq_entry *cqe)
826 {
827 if (cqe)
828 return le32_to_cpu(cqe->status) & MASK_OCS;
829
830 return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS;
831 }
832
833 /**
834 * ufshcd_utrl_clear() - Clear requests from the controller request list.
835 * @hba: per adapter instance
836 * @mask: mask with one bit set for each request to be cleared
837 */
ufshcd_utrl_clear(struct ufs_hba * hba,u32 mask)838 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask)
839 {
840 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
841 mask = ~mask;
842 /*
843 * From the UFSHCI specification: "UTP Transfer Request List CLear
844 * Register (UTRLCLR): This field is bit significant. Each bit
845 * corresponds to a slot in the UTP Transfer Request List, where bit 0
846 * corresponds to request slot 0. A bit in this field is set to ‘0’
847 * by host software to indicate to the host controller that a transfer
848 * request slot is cleared. The host controller
849 * shall free up any resources associated to the request slot
850 * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The
851 * host software indicates no change to request slots by setting the
852 * associated bits in this field to ‘1’. Bits in this field shall only
853 * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’."
854 */
855 ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR);
856 }
857
858 /**
859 * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register
860 * @hba: per adapter instance
861 * @pos: position of the bit to be cleared
862 */
ufshcd_utmrl_clear(struct ufs_hba * hba,u32 pos)863 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
864 {
865 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
866 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
867 else
868 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
869 }
870
871 /**
872 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
873 * @reg: Register value of host controller status
874 *
875 * Return: 0 on success; a positive value if failed.
876 */
ufshcd_get_lists_status(u32 reg)877 static inline int ufshcd_get_lists_status(u32 reg)
878 {
879 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
880 }
881
882 /**
883 * ufshcd_get_uic_cmd_result - Get the UIC command result
884 * @hba: Pointer to adapter instance
885 *
886 * This function gets the result of UIC command completion
887 *
888 * Return: 0 on success; non-zero value on error.
889 */
ufshcd_get_uic_cmd_result(struct ufs_hba * hba)890 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
891 {
892 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
893 MASK_UIC_COMMAND_RESULT;
894 }
895
896 /**
897 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
898 * @hba: Pointer to adapter instance
899 *
900 * This function gets UIC command argument3
901 *
902 * Return: 0 on success; non-zero value on error.
903 */
ufshcd_get_dme_attr_val(struct ufs_hba * hba)904 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
905 {
906 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
907 }
908
909 /**
910 * ufshcd_get_req_rsp - returns the TR response transaction type
911 * @ucd_rsp_ptr: pointer to response UPIU
912 *
913 * Return: UPIU type.
914 */
915 static inline enum upiu_response_transaction
ufshcd_get_req_rsp(struct utp_upiu_rsp * ucd_rsp_ptr)916 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
917 {
918 return ucd_rsp_ptr->header.transaction_code;
919 }
920
921 /**
922 * ufshcd_is_exception_event - Check if the device raised an exception event
923 * @ucd_rsp_ptr: pointer to response UPIU
924 *
925 * The function checks if the device raised an exception event indicated in
926 * the Device Information field of response UPIU.
927 *
928 * Return: true if exception is raised, false otherwise.
929 */
ufshcd_is_exception_event(struct utp_upiu_rsp * ucd_rsp_ptr)930 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
931 {
932 return ucd_rsp_ptr->header.device_information & 1;
933 }
934
935 /**
936 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
937 * @hba: per adapter instance
938 */
939 static inline void
ufshcd_reset_intr_aggr(struct ufs_hba * hba)940 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
941 {
942 ufshcd_writel(hba, INT_AGGR_ENABLE |
943 INT_AGGR_COUNTER_AND_TIMER_RESET,
944 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
945 }
946
947 /**
948 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
949 * @hba: per adapter instance
950 * @cnt: Interrupt aggregation counter threshold
951 * @tmout: Interrupt aggregation timeout value
952 */
953 static inline void
ufshcd_config_intr_aggr(struct ufs_hba * hba,u8 cnt,u8 tmout)954 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
955 {
956 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
957 INT_AGGR_COUNTER_THLD_VAL(cnt) |
958 INT_AGGR_TIMEOUT_VAL(tmout),
959 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
960 }
961
962 /**
963 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
964 * @hba: per adapter instance
965 */
ufshcd_disable_intr_aggr(struct ufs_hba * hba)966 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
967 {
968 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
969 }
970
971 /**
972 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
973 * When run-stop registers are set to 1, it indicates the
974 * host controller that it can process the requests
975 * @hba: per adapter instance
976 */
ufshcd_enable_run_stop_reg(struct ufs_hba * hba)977 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
978 {
979 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
980 REG_UTP_TASK_REQ_LIST_RUN_STOP);
981 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
982 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
983 }
984
985 /**
986 * ufshcd_hba_start - Start controller initialization sequence
987 * @hba: per adapter instance
988 */
ufshcd_hba_start(struct ufs_hba * hba)989 static inline void ufshcd_hba_start(struct ufs_hba *hba)
990 {
991 u32 val = CONTROLLER_ENABLE;
992
993 if (ufshcd_crypto_enable(hba))
994 val |= CRYPTO_GENERAL_ENABLE;
995
996 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
997 }
998
999 /**
1000 * ufshcd_is_hba_active - Get controller state
1001 * @hba: per adapter instance
1002 *
1003 * Return: true if and only if the controller is active.
1004 */
ufshcd_is_hba_active(struct ufs_hba * hba)1005 bool ufshcd_is_hba_active(struct ufs_hba *hba)
1006 {
1007 return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE;
1008 }
1009 EXPORT_SYMBOL_GPL(ufshcd_is_hba_active);
1010
1011 /**
1012 * ufshcd_pm_qos_init - initialize PM QoS request
1013 * @hba: per adapter instance
1014 */
ufshcd_pm_qos_init(struct ufs_hba * hba)1015 void ufshcd_pm_qos_init(struct ufs_hba *hba)
1016 {
1017
1018 if (hba->pm_qos_enabled)
1019 return;
1020
1021 cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE);
1022
1023 if (cpu_latency_qos_request_active(&hba->pm_qos_req))
1024 hba->pm_qos_enabled = true;
1025 }
1026
1027 /**
1028 * ufshcd_pm_qos_exit - remove request from PM QoS
1029 * @hba: per adapter instance
1030 */
ufshcd_pm_qos_exit(struct ufs_hba * hba)1031 void ufshcd_pm_qos_exit(struct ufs_hba *hba)
1032 {
1033 if (!hba->pm_qos_enabled)
1034 return;
1035
1036 cpu_latency_qos_remove_request(&hba->pm_qos_req);
1037 hba->pm_qos_enabled = false;
1038 }
1039
1040 /**
1041 * ufshcd_pm_qos_update - update PM QoS request
1042 * @hba: per adapter instance
1043 * @on: If True, vote for perf PM QoS mode otherwise power save mode
1044 */
ufshcd_pm_qos_update(struct ufs_hba * hba,bool on)1045 static void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on)
1046 {
1047 if (!hba->pm_qos_enabled)
1048 return;
1049
1050 cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE);
1051 }
1052
1053 /**
1054 * ufshcd_set_clk_freq - set UFS controller clock frequencies
1055 * @hba: per adapter instance
1056 * @scale_up: If True, set max possible frequency othewise set low frequency
1057 *
1058 * Return: 0 if successful; < 0 upon failure.
1059 */
ufshcd_set_clk_freq(struct ufs_hba * hba,bool scale_up)1060 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
1061 {
1062 int ret = 0;
1063 struct ufs_clk_info *clki;
1064 struct list_head *head = &hba->clk_list_head;
1065
1066 if (list_empty(head))
1067 goto out;
1068
1069 list_for_each_entry(clki, head, list) {
1070 if (!IS_ERR_OR_NULL(clki->clk)) {
1071 if (scale_up && clki->max_freq) {
1072 if (clki->curr_freq == clki->max_freq)
1073 continue;
1074
1075 ret = clk_set_rate(clki->clk, clki->max_freq);
1076 if (ret) {
1077 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1078 __func__, clki->name,
1079 clki->max_freq, ret);
1080 break;
1081 }
1082 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1083 "scaled up", clki->name,
1084 clki->curr_freq,
1085 clki->max_freq);
1086
1087 clki->curr_freq = clki->max_freq;
1088
1089 } else if (!scale_up && clki->min_freq) {
1090 if (clki->curr_freq == clki->min_freq)
1091 continue;
1092
1093 ret = clk_set_rate(clki->clk, clki->min_freq);
1094 if (ret) {
1095 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1096 __func__, clki->name,
1097 clki->min_freq, ret);
1098 break;
1099 }
1100 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1101 "scaled down", clki->name,
1102 clki->curr_freq,
1103 clki->min_freq);
1104 clki->curr_freq = clki->min_freq;
1105 }
1106 }
1107 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1108 clki->name, clk_get_rate(clki->clk));
1109 }
1110
1111 out:
1112 return ret;
1113 }
1114
ufshcd_opp_config_clks(struct device * dev,struct opp_table * opp_table,struct dev_pm_opp * opp,void * data,bool scaling_down)1115 int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table,
1116 struct dev_pm_opp *opp, void *data,
1117 bool scaling_down)
1118 {
1119 struct ufs_hba *hba = dev_get_drvdata(dev);
1120 struct list_head *head = &hba->clk_list_head;
1121 struct ufs_clk_info *clki;
1122 unsigned long freq;
1123 u8 idx = 0;
1124 int ret;
1125
1126 list_for_each_entry(clki, head, list) {
1127 if (!IS_ERR_OR_NULL(clki->clk)) {
1128 freq = dev_pm_opp_get_freq_indexed(opp, idx++);
1129
1130 /* Do not set rate for clocks having frequency as 0 */
1131 if (!freq)
1132 continue;
1133
1134 ret = clk_set_rate(clki->clk, freq);
1135 if (ret) {
1136 dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n",
1137 __func__, clki->name, freq, ret);
1138 return ret;
1139 }
1140
1141 trace_ufshcd_clk_scaling(dev_name(dev),
1142 (scaling_down ? "scaled down" : "scaled up"),
1143 clki->name, hba->clk_scaling.target_freq, freq);
1144 }
1145 }
1146
1147 return 0;
1148 }
1149 EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks);
1150
ufshcd_opp_set_rate(struct ufs_hba * hba,unsigned long freq)1151 static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq)
1152 {
1153 struct dev_pm_opp *opp;
1154 int ret;
1155
1156 opp = dev_pm_opp_find_freq_floor_indexed(hba->dev,
1157 &freq, 0);
1158 if (IS_ERR(opp))
1159 return PTR_ERR(opp);
1160
1161 ret = dev_pm_opp_set_opp(hba->dev, opp);
1162 dev_pm_opp_put(opp);
1163
1164 return ret;
1165 }
1166
1167 /**
1168 * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1169 * @hba: per adapter instance
1170 * @freq: frequency to scale
1171 * @scale_up: True if scaling up and false if scaling down
1172 *
1173 * Return: 0 if successful; < 0 upon failure.
1174 */
ufshcd_scale_clks(struct ufs_hba * hba,unsigned long freq,bool scale_up)1175 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq,
1176 bool scale_up)
1177 {
1178 int ret = 0;
1179 ktime_t start = ktime_get();
1180
1181 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1182 if (ret)
1183 goto out;
1184
1185 if (hba->use_pm_opp)
1186 ret = ufshcd_opp_set_rate(hba, freq);
1187 else
1188 ret = ufshcd_set_clk_freq(hba, scale_up);
1189 if (ret)
1190 goto out;
1191
1192 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1193 if (ret) {
1194 if (hba->use_pm_opp)
1195 ufshcd_opp_set_rate(hba,
1196 hba->devfreq->previous_freq);
1197 else
1198 ufshcd_set_clk_freq(hba, !scale_up);
1199 goto out;
1200 }
1201
1202 ufshcd_pm_qos_update(hba, scale_up);
1203
1204 out:
1205 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1206 (scale_up ? "up" : "down"),
1207 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1208 return ret;
1209 }
1210
1211 /**
1212 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1213 * @hba: per adapter instance
1214 * @freq: frequency to scale
1215 * @scale_up: True if scaling up and false if scaling down
1216 *
1217 * Return: true if scaling is required, false otherwise.
1218 */
ufshcd_is_devfreq_scaling_required(struct ufs_hba * hba,unsigned long freq,bool scale_up)1219 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1220 unsigned long freq, bool scale_up)
1221 {
1222 struct ufs_clk_info *clki;
1223 struct list_head *head = &hba->clk_list_head;
1224
1225 if (list_empty(head))
1226 return false;
1227
1228 if (hba->use_pm_opp)
1229 return freq != hba->clk_scaling.target_freq;
1230
1231 list_for_each_entry(clki, head, list) {
1232 if (!IS_ERR_OR_NULL(clki->clk)) {
1233 if (scale_up && clki->max_freq) {
1234 if (clki->curr_freq == clki->max_freq)
1235 continue;
1236 return true;
1237 } else if (!scale_up && clki->min_freq) {
1238 if (clki->curr_freq == clki->min_freq)
1239 continue;
1240 return true;
1241 }
1242 }
1243 }
1244
1245 return false;
1246 }
1247
1248 /*
1249 * Determine the number of pending commands by counting the bits in the SCSI
1250 * device budget maps. This approach has been selected because a bit is set in
1251 * the budget map before scsi_host_queue_ready() checks the host_self_blocked
1252 * flag. The host_self_blocked flag can be modified by calling
1253 * scsi_block_requests() or scsi_unblock_requests().
1254 */
ufshcd_pending_cmds(struct ufs_hba * hba)1255 static u32 ufshcd_pending_cmds(struct ufs_hba *hba)
1256 {
1257 const struct scsi_device *sdev;
1258 u32 pending = 0;
1259
1260 lockdep_assert_held(hba->host->host_lock);
1261 __shost_for_each_device(sdev, hba->host)
1262 pending += sbitmap_weight(&sdev->budget_map);
1263
1264 return pending;
1265 }
1266
1267 /*
1268 * Wait until all pending SCSI commands and TMFs have finished or the timeout
1269 * has expired.
1270 *
1271 * Return: 0 upon success; -EBUSY upon timeout.
1272 */
ufshcd_wait_for_doorbell_clr(struct ufs_hba * hba,u64 wait_timeout_us)1273 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1274 u64 wait_timeout_us)
1275 {
1276 unsigned long flags;
1277 int ret = 0;
1278 u32 tm_doorbell;
1279 u32 tr_pending;
1280 bool timeout = false, do_last_check = false;
1281 ktime_t start;
1282
1283 ufshcd_hold(hba);
1284 spin_lock_irqsave(hba->host->host_lock, flags);
1285 /*
1286 * Wait for all the outstanding tasks/transfer requests.
1287 * Verify by checking the doorbell registers are clear.
1288 */
1289 start = ktime_get();
1290 do {
1291 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1292 ret = -EBUSY;
1293 goto out;
1294 }
1295
1296 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1297 tr_pending = ufshcd_pending_cmds(hba);
1298 if (!tm_doorbell && !tr_pending) {
1299 timeout = false;
1300 break;
1301 } else if (do_last_check) {
1302 break;
1303 }
1304
1305 spin_unlock_irqrestore(hba->host->host_lock, flags);
1306 io_schedule_timeout(msecs_to_jiffies(20));
1307 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1308 wait_timeout_us) {
1309 timeout = true;
1310 /*
1311 * We might have scheduled out for long time so make
1312 * sure to check if doorbells are cleared by this time
1313 * or not.
1314 */
1315 do_last_check = true;
1316 }
1317 spin_lock_irqsave(hba->host->host_lock, flags);
1318 } while (tm_doorbell || tr_pending);
1319
1320 if (timeout) {
1321 dev_err(hba->dev,
1322 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1323 __func__, tm_doorbell, tr_pending);
1324 ret = -EBUSY;
1325 }
1326 out:
1327 spin_unlock_irqrestore(hba->host->host_lock, flags);
1328 ufshcd_release(hba);
1329 return ret;
1330 }
1331
1332 /**
1333 * ufshcd_scale_gear - scale up/down UFS gear
1334 * @hba: per adapter instance
1335 * @scale_up: True for scaling up gear and false for scaling down
1336 *
1337 * Return: 0 for success; -EBUSY if scaling can't happen at this time;
1338 * non-zero for any other errors.
1339 */
ufshcd_scale_gear(struct ufs_hba * hba,bool scale_up)1340 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1341 {
1342 int ret = 0;
1343 struct ufs_pa_layer_attr new_pwr_info;
1344
1345 if (scale_up) {
1346 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info,
1347 sizeof(struct ufs_pa_layer_attr));
1348 } else {
1349 memcpy(&new_pwr_info, &hba->pwr_info,
1350 sizeof(struct ufs_pa_layer_attr));
1351
1352 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1353 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1354 /* save the current power mode */
1355 memcpy(&hba->clk_scaling.saved_pwr_info,
1356 &hba->pwr_info,
1357 sizeof(struct ufs_pa_layer_attr));
1358
1359 /* scale down gear */
1360 new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1361 new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1362 }
1363 }
1364
1365 /* check if the power mode needs to be changed or not? */
1366 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1367 if (ret)
1368 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1369 __func__, ret,
1370 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1371 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1372
1373 return ret;
1374 }
1375
1376 /*
1377 * Wait until all pending SCSI commands and TMFs have finished or the timeout
1378 * has expired.
1379 *
1380 * Return: 0 upon success; -EBUSY upon timeout.
1381 */
ufshcd_clock_scaling_prepare(struct ufs_hba * hba,u64 timeout_us)1382 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us)
1383 {
1384 int ret = 0;
1385 /*
1386 * make sure that there are no outstanding requests when
1387 * clock scaling is in progress
1388 */
1389 blk_mq_quiesce_tagset(&hba->host->tag_set);
1390 mutex_lock(&hba->wb_mutex);
1391 down_write(&hba->clk_scaling_lock);
1392
1393 if (!hba->clk_scaling.is_allowed ||
1394 ufshcd_wait_for_doorbell_clr(hba, timeout_us)) {
1395 ret = -EBUSY;
1396 up_write(&hba->clk_scaling_lock);
1397 mutex_unlock(&hba->wb_mutex);
1398 blk_mq_unquiesce_tagset(&hba->host->tag_set);
1399 goto out;
1400 }
1401
1402 /* let's not get into low power until clock scaling is completed */
1403 ufshcd_hold(hba);
1404
1405 out:
1406 return ret;
1407 }
1408
ufshcd_clock_scaling_unprepare(struct ufs_hba * hba,int err,bool scale_up)1409 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool scale_up)
1410 {
1411 up_write(&hba->clk_scaling_lock);
1412
1413 /* Enable Write Booster if we have scaled up else disable it */
1414 if (ufshcd_enable_wb_if_scaling_up(hba) && !err)
1415 ufshcd_wb_toggle(hba, scale_up);
1416
1417 mutex_unlock(&hba->wb_mutex);
1418
1419 blk_mq_unquiesce_tagset(&hba->host->tag_set);
1420 ufshcd_release(hba);
1421 }
1422
1423 /**
1424 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1425 * @hba: per adapter instance
1426 * @freq: frequency to scale
1427 * @scale_up: True for scaling up and false for scalin down
1428 *
1429 * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero
1430 * for any other errors.
1431 */
ufshcd_devfreq_scale(struct ufs_hba * hba,unsigned long freq,bool scale_up)1432 static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq,
1433 bool scale_up)
1434 {
1435 int ret = 0;
1436
1437 ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC);
1438 if (ret)
1439 return ret;
1440
1441 /* scale down the gear before scaling down clocks */
1442 if (!scale_up) {
1443 ret = ufshcd_scale_gear(hba, false);
1444 if (ret)
1445 goto out_unprepare;
1446 }
1447
1448 ret = ufshcd_scale_clks(hba, freq, scale_up);
1449 if (ret) {
1450 if (!scale_up)
1451 ufshcd_scale_gear(hba, true);
1452 goto out_unprepare;
1453 }
1454
1455 /* scale up the gear after scaling up clocks */
1456 if (scale_up) {
1457 ret = ufshcd_scale_gear(hba, true);
1458 if (ret) {
1459 ufshcd_scale_clks(hba, hba->devfreq->previous_freq,
1460 false);
1461 goto out_unprepare;
1462 }
1463 }
1464
1465 out_unprepare:
1466 ufshcd_clock_scaling_unprepare(hba, ret, scale_up);
1467 return ret;
1468 }
1469
ufshcd_clk_scaling_suspend_work(struct work_struct * work)1470 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1471 {
1472 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1473 clk_scaling.suspend_work);
1474 unsigned long irq_flags;
1475
1476 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1477 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1478 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1479 return;
1480 }
1481 hba->clk_scaling.is_suspended = true;
1482 hba->clk_scaling.window_start_t = 0;
1483 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1484
1485 devfreq_suspend_device(hba->devfreq);
1486 }
1487
ufshcd_clk_scaling_resume_work(struct work_struct * work)1488 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1489 {
1490 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1491 clk_scaling.resume_work);
1492 unsigned long irq_flags;
1493
1494 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1495 if (!hba->clk_scaling.is_suspended) {
1496 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1497 return;
1498 }
1499 hba->clk_scaling.is_suspended = false;
1500 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1501
1502 devfreq_resume_device(hba->devfreq);
1503 }
1504
ufshcd_devfreq_target(struct device * dev,unsigned long * freq,u32 flags)1505 static int ufshcd_devfreq_target(struct device *dev,
1506 unsigned long *freq, u32 flags)
1507 {
1508 int ret = 0;
1509 struct ufs_hba *hba = dev_get_drvdata(dev);
1510 ktime_t start;
1511 bool scale_up = false, sched_clk_scaling_suspend_work = false;
1512 struct list_head *clk_list = &hba->clk_list_head;
1513 struct ufs_clk_info *clki;
1514 unsigned long irq_flags;
1515
1516 if (!ufshcd_is_clkscaling_supported(hba))
1517 return -EINVAL;
1518
1519 if (hba->use_pm_opp) {
1520 struct dev_pm_opp *opp;
1521
1522 /* Get the recommended frequency from OPP framework */
1523 opp = devfreq_recommended_opp(dev, freq, flags);
1524 if (IS_ERR(opp))
1525 return PTR_ERR(opp);
1526
1527 dev_pm_opp_put(opp);
1528 } else {
1529 /* Override with the closest supported frequency */
1530 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info,
1531 list);
1532 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1533 }
1534
1535 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1536 if (ufshcd_eh_in_progress(hba)) {
1537 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1538 return 0;
1539 }
1540
1541 /* Skip scaling clock when clock scaling is suspended */
1542 if (hba->clk_scaling.is_suspended) {
1543 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1544 dev_warn(hba->dev, "clock scaling is suspended, skip");
1545 return 0;
1546 }
1547
1548 if (!hba->clk_scaling.active_reqs)
1549 sched_clk_scaling_suspend_work = true;
1550
1551 if (list_empty(clk_list)) {
1552 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1553 goto out;
1554 }
1555
1556 /* Decide based on the target or rounded-off frequency and update */
1557 if (hba->use_pm_opp)
1558 scale_up = *freq > hba->clk_scaling.target_freq;
1559 else
1560 scale_up = *freq == clki->max_freq;
1561
1562 if (!hba->use_pm_opp && !scale_up)
1563 *freq = clki->min_freq;
1564
1565 /* Update the frequency */
1566 if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) {
1567 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1568 ret = 0;
1569 goto out; /* no state change required */
1570 }
1571 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1572
1573 start = ktime_get();
1574 ret = ufshcd_devfreq_scale(hba, *freq, scale_up);
1575 if (!ret)
1576 hba->clk_scaling.target_freq = *freq;
1577
1578 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1579 (scale_up ? "up" : "down"),
1580 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1581
1582 out:
1583 if (sched_clk_scaling_suspend_work &&
1584 (!scale_up || hba->clk_scaling.suspend_on_no_request))
1585 queue_work(hba->clk_scaling.workq,
1586 &hba->clk_scaling.suspend_work);
1587
1588 return ret;
1589 }
1590
ufshcd_devfreq_get_dev_status(struct device * dev,struct devfreq_dev_status * stat)1591 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1592 struct devfreq_dev_status *stat)
1593 {
1594 struct ufs_hba *hba = dev_get_drvdata(dev);
1595 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1596 unsigned long flags;
1597 ktime_t curr_t;
1598
1599 if (!ufshcd_is_clkscaling_supported(hba))
1600 return -EINVAL;
1601
1602 memset(stat, 0, sizeof(*stat));
1603
1604 spin_lock_irqsave(hba->host->host_lock, flags);
1605 curr_t = ktime_get();
1606 if (!scaling->window_start_t)
1607 goto start_window;
1608
1609 /*
1610 * If current frequency is 0, then the ondemand governor considers
1611 * there's no initial frequency set. And it always requests to set
1612 * to max. frequency.
1613 */
1614 if (hba->use_pm_opp) {
1615 stat->current_frequency = hba->clk_scaling.target_freq;
1616 } else {
1617 struct list_head *clk_list = &hba->clk_list_head;
1618 struct ufs_clk_info *clki;
1619
1620 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1621 stat->current_frequency = clki->curr_freq;
1622 }
1623
1624 if (scaling->is_busy_started)
1625 scaling->tot_busy_t += ktime_us_delta(curr_t,
1626 scaling->busy_start_t);
1627 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1628 stat->busy_time = scaling->tot_busy_t;
1629 start_window:
1630 scaling->window_start_t = curr_t;
1631 scaling->tot_busy_t = 0;
1632
1633 if (scaling->active_reqs) {
1634 scaling->busy_start_t = curr_t;
1635 scaling->is_busy_started = true;
1636 } else {
1637 scaling->busy_start_t = 0;
1638 scaling->is_busy_started = false;
1639 }
1640 spin_unlock_irqrestore(hba->host->host_lock, flags);
1641 return 0;
1642 }
1643
ufshcd_devfreq_init(struct ufs_hba * hba)1644 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1645 {
1646 struct list_head *clk_list = &hba->clk_list_head;
1647 struct ufs_clk_info *clki;
1648 struct devfreq *devfreq;
1649 int ret;
1650
1651 /* Skip devfreq if we don't have any clocks in the list */
1652 if (list_empty(clk_list))
1653 return 0;
1654
1655 if (!hba->use_pm_opp) {
1656 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1657 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1658 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1659 }
1660
1661 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1662 &hba->vps->ondemand_data);
1663 devfreq = devfreq_add_device(hba->dev,
1664 &hba->vps->devfreq_profile,
1665 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1666 &hba->vps->ondemand_data);
1667 if (IS_ERR(devfreq)) {
1668 ret = PTR_ERR(devfreq);
1669 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1670
1671 if (!hba->use_pm_opp) {
1672 dev_pm_opp_remove(hba->dev, clki->min_freq);
1673 dev_pm_opp_remove(hba->dev, clki->max_freq);
1674 }
1675 return ret;
1676 }
1677
1678 hba->devfreq = devfreq;
1679
1680 return 0;
1681 }
1682
ufshcd_devfreq_remove(struct ufs_hba * hba)1683 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1684 {
1685 struct list_head *clk_list = &hba->clk_list_head;
1686
1687 if (!hba->devfreq)
1688 return;
1689
1690 devfreq_remove_device(hba->devfreq);
1691 hba->devfreq = NULL;
1692
1693 if (!hba->use_pm_opp) {
1694 struct ufs_clk_info *clki;
1695
1696 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1697 dev_pm_opp_remove(hba->dev, clki->min_freq);
1698 dev_pm_opp_remove(hba->dev, clki->max_freq);
1699 }
1700 }
1701
ufshcd_suspend_clkscaling(struct ufs_hba * hba)1702 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1703 {
1704 unsigned long flags;
1705 bool suspend = false;
1706
1707 cancel_work_sync(&hba->clk_scaling.suspend_work);
1708 cancel_work_sync(&hba->clk_scaling.resume_work);
1709
1710 spin_lock_irqsave(hba->host->host_lock, flags);
1711 if (!hba->clk_scaling.is_suspended) {
1712 suspend = true;
1713 hba->clk_scaling.is_suspended = true;
1714 hba->clk_scaling.window_start_t = 0;
1715 }
1716 spin_unlock_irqrestore(hba->host->host_lock, flags);
1717
1718 if (suspend)
1719 devfreq_suspend_device(hba->devfreq);
1720 }
1721
ufshcd_resume_clkscaling(struct ufs_hba * hba)1722 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1723 {
1724 unsigned long flags;
1725 bool resume = false;
1726
1727 spin_lock_irqsave(hba->host->host_lock, flags);
1728 if (hba->clk_scaling.is_suspended) {
1729 resume = true;
1730 hba->clk_scaling.is_suspended = false;
1731 }
1732 spin_unlock_irqrestore(hba->host->host_lock, flags);
1733
1734 if (resume)
1735 devfreq_resume_device(hba->devfreq);
1736 }
1737
ufshcd_clkscale_enable_show(struct device * dev,struct device_attribute * attr,char * buf)1738 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1739 struct device_attribute *attr, char *buf)
1740 {
1741 struct ufs_hba *hba = dev_get_drvdata(dev);
1742
1743 return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
1744 }
1745
ufshcd_clkscale_enable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1746 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1747 struct device_attribute *attr, const char *buf, size_t count)
1748 {
1749 struct ufs_hba *hba = dev_get_drvdata(dev);
1750 u32 value;
1751 int err = 0;
1752
1753 if (kstrtou32(buf, 0, &value))
1754 return -EINVAL;
1755
1756 down(&hba->host_sem);
1757 if (!ufshcd_is_user_access_allowed(hba)) {
1758 err = -EBUSY;
1759 goto out;
1760 }
1761
1762 value = !!value;
1763 if (value == hba->clk_scaling.is_enabled)
1764 goto out;
1765
1766 ufshcd_rpm_get_sync(hba);
1767 ufshcd_hold(hba);
1768
1769 hba->clk_scaling.is_enabled = value;
1770
1771 if (value) {
1772 ufshcd_resume_clkscaling(hba);
1773 } else {
1774 ufshcd_suspend_clkscaling(hba);
1775 err = ufshcd_devfreq_scale(hba, ULONG_MAX, true);
1776 if (err)
1777 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1778 __func__, err);
1779 }
1780
1781 ufshcd_release(hba);
1782 ufshcd_rpm_put_sync(hba);
1783 out:
1784 up(&hba->host_sem);
1785 return err ? err : count;
1786 }
1787
ufshcd_init_clk_scaling_sysfs(struct ufs_hba * hba)1788 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
1789 {
1790 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1791 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1792 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1793 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1794 hba->clk_scaling.enable_attr.attr.mode = 0644;
1795 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1796 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1797 }
1798
ufshcd_remove_clk_scaling_sysfs(struct ufs_hba * hba)1799 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1800 {
1801 if (hba->clk_scaling.enable_attr.attr.name)
1802 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1803 }
1804
ufshcd_init_clk_scaling(struct ufs_hba * hba)1805 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1806 {
1807 if (!ufshcd_is_clkscaling_supported(hba))
1808 return;
1809
1810 if (!hba->clk_scaling.min_gear)
1811 hba->clk_scaling.min_gear = UFS_HS_G1;
1812
1813 INIT_WORK(&hba->clk_scaling.suspend_work,
1814 ufshcd_clk_scaling_suspend_work);
1815 INIT_WORK(&hba->clk_scaling.resume_work,
1816 ufshcd_clk_scaling_resume_work);
1817
1818 hba->clk_scaling.workq = alloc_ordered_workqueue(
1819 "ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no);
1820
1821 hba->clk_scaling.is_initialized = true;
1822 }
1823
ufshcd_exit_clk_scaling(struct ufs_hba * hba)1824 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1825 {
1826 if (!hba->clk_scaling.is_initialized)
1827 return;
1828
1829 ufshcd_remove_clk_scaling_sysfs(hba);
1830 destroy_workqueue(hba->clk_scaling.workq);
1831 ufshcd_devfreq_remove(hba);
1832 hba->clk_scaling.is_initialized = false;
1833 }
1834
ufshcd_ungate_work(struct work_struct * work)1835 static void ufshcd_ungate_work(struct work_struct *work)
1836 {
1837 int ret;
1838 unsigned long flags;
1839 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1840 clk_gating.ungate_work);
1841
1842 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1843
1844 spin_lock_irqsave(hba->host->host_lock, flags);
1845 if (hba->clk_gating.state == CLKS_ON) {
1846 spin_unlock_irqrestore(hba->host->host_lock, flags);
1847 return;
1848 }
1849
1850 spin_unlock_irqrestore(hba->host->host_lock, flags);
1851 ufshcd_hba_vreg_set_hpm(hba);
1852 ufshcd_setup_clocks(hba, true);
1853
1854 ufshcd_enable_irq(hba);
1855
1856 /* Exit from hibern8 */
1857 if (ufshcd_can_hibern8_during_gating(hba)) {
1858 /* Prevent gating in this path */
1859 hba->clk_gating.is_suspended = true;
1860 if (ufshcd_is_link_hibern8(hba)) {
1861 ret = ufshcd_uic_hibern8_exit(hba);
1862 if (ret)
1863 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1864 __func__, ret);
1865 else
1866 ufshcd_set_link_active(hba);
1867 }
1868 hba->clk_gating.is_suspended = false;
1869 }
1870 }
1871
1872 /**
1873 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1874 * Also, exit from hibern8 mode and set the link as active.
1875 * @hba: per adapter instance
1876 */
ufshcd_hold(struct ufs_hba * hba)1877 void ufshcd_hold(struct ufs_hba *hba)
1878 {
1879 bool flush_result;
1880 unsigned long flags;
1881
1882 if (!ufshcd_is_clkgating_allowed(hba) ||
1883 !hba->clk_gating.is_initialized)
1884 return;
1885 spin_lock_irqsave(hba->host->host_lock, flags);
1886 hba->clk_gating.active_reqs++;
1887
1888 start:
1889 switch (hba->clk_gating.state) {
1890 case CLKS_ON:
1891 /*
1892 * Wait for the ungate work to complete if in progress.
1893 * Though the clocks may be in ON state, the link could
1894 * still be in hibner8 state if hibern8 is allowed
1895 * during clock gating.
1896 * Make sure we exit hibern8 state also in addition to
1897 * clocks being ON.
1898 */
1899 if (ufshcd_can_hibern8_during_gating(hba) &&
1900 ufshcd_is_link_hibern8(hba)) {
1901 spin_unlock_irqrestore(hba->host->host_lock, flags);
1902 flush_result = flush_work(&hba->clk_gating.ungate_work);
1903 if (hba->clk_gating.is_suspended && !flush_result)
1904 return;
1905 spin_lock_irqsave(hba->host->host_lock, flags);
1906 goto start;
1907 }
1908 break;
1909 case REQ_CLKS_OFF:
1910 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1911 hba->clk_gating.state = CLKS_ON;
1912 trace_ufshcd_clk_gating(dev_name(hba->dev),
1913 hba->clk_gating.state);
1914 break;
1915 }
1916 /*
1917 * If we are here, it means gating work is either done or
1918 * currently running. Hence, fall through to cancel gating
1919 * work and to enable clocks.
1920 */
1921 fallthrough;
1922 case CLKS_OFF:
1923 hba->clk_gating.state = REQ_CLKS_ON;
1924 trace_ufshcd_clk_gating(dev_name(hba->dev),
1925 hba->clk_gating.state);
1926 queue_work(hba->clk_gating.clk_gating_workq,
1927 &hba->clk_gating.ungate_work);
1928 /*
1929 * fall through to check if we should wait for this
1930 * work to be done or not.
1931 */
1932 fallthrough;
1933 case REQ_CLKS_ON:
1934 spin_unlock_irqrestore(hba->host->host_lock, flags);
1935 flush_work(&hba->clk_gating.ungate_work);
1936 /* Make sure state is CLKS_ON before returning */
1937 spin_lock_irqsave(hba->host->host_lock, flags);
1938 goto start;
1939 default:
1940 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1941 __func__, hba->clk_gating.state);
1942 break;
1943 }
1944 spin_unlock_irqrestore(hba->host->host_lock, flags);
1945 }
1946 EXPORT_SYMBOL_GPL(ufshcd_hold);
1947
ufshcd_gate_work(struct work_struct * work)1948 static void ufshcd_gate_work(struct work_struct *work)
1949 {
1950 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1951 clk_gating.gate_work.work);
1952 unsigned long flags;
1953 int ret;
1954
1955 spin_lock_irqsave(hba->host->host_lock, flags);
1956 /*
1957 * In case you are here to cancel this work the gating state
1958 * would be marked as REQ_CLKS_ON. In this case save time by
1959 * skipping the gating work and exit after changing the clock
1960 * state to CLKS_ON.
1961 */
1962 if (hba->clk_gating.is_suspended ||
1963 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1964 hba->clk_gating.state = CLKS_ON;
1965 trace_ufshcd_clk_gating(dev_name(hba->dev),
1966 hba->clk_gating.state);
1967 goto rel_lock;
1968 }
1969
1970 if (ufshcd_is_ufs_dev_busy(hba) || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
1971 goto rel_lock;
1972
1973 spin_unlock_irqrestore(hba->host->host_lock, flags);
1974
1975 /* put the link into hibern8 mode before turning off clocks */
1976 if (ufshcd_can_hibern8_during_gating(hba)) {
1977 ret = ufshcd_uic_hibern8_enter(hba);
1978 if (ret) {
1979 hba->clk_gating.state = CLKS_ON;
1980 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1981 __func__, ret);
1982 trace_ufshcd_clk_gating(dev_name(hba->dev),
1983 hba->clk_gating.state);
1984 goto out;
1985 }
1986 ufshcd_set_link_hibern8(hba);
1987 }
1988
1989 ufshcd_disable_irq(hba);
1990
1991 ufshcd_setup_clocks(hba, false);
1992
1993 /* Put the host controller in low power mode if possible */
1994 ufshcd_hba_vreg_set_lpm(hba);
1995 /*
1996 * In case you are here to cancel this work the gating state
1997 * would be marked as REQ_CLKS_ON. In this case keep the state
1998 * as REQ_CLKS_ON which would anyway imply that clocks are off
1999 * and a request to turn them on is pending. By doing this way,
2000 * we keep the state machine in tact and this would ultimately
2001 * prevent from doing cancel work multiple times when there are
2002 * new requests arriving before the current cancel work is done.
2003 */
2004 spin_lock_irqsave(hba->host->host_lock, flags);
2005 if (hba->clk_gating.state == REQ_CLKS_OFF) {
2006 hba->clk_gating.state = CLKS_OFF;
2007 trace_ufshcd_clk_gating(dev_name(hba->dev),
2008 hba->clk_gating.state);
2009 }
2010 rel_lock:
2011 spin_unlock_irqrestore(hba->host->host_lock, flags);
2012 out:
2013 return;
2014 }
2015
2016 /* host lock must be held before calling this variant */
__ufshcd_release(struct ufs_hba * hba)2017 static void __ufshcd_release(struct ufs_hba *hba)
2018 {
2019 if (!ufshcd_is_clkgating_allowed(hba))
2020 return;
2021
2022 hba->clk_gating.active_reqs--;
2023
2024 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
2025 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
2026 hba->outstanding_tasks || !hba->clk_gating.is_initialized ||
2027 hba->active_uic_cmd || hba->uic_async_done ||
2028 hba->clk_gating.state == CLKS_OFF)
2029 return;
2030
2031 hba->clk_gating.state = REQ_CLKS_OFF;
2032 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
2033 queue_delayed_work(hba->clk_gating.clk_gating_workq,
2034 &hba->clk_gating.gate_work,
2035 msecs_to_jiffies(hba->clk_gating.delay_ms));
2036 }
2037
ufshcd_release(struct ufs_hba * hba)2038 void ufshcd_release(struct ufs_hba *hba)
2039 {
2040 unsigned long flags;
2041
2042 spin_lock_irqsave(hba->host->host_lock, flags);
2043 __ufshcd_release(hba);
2044 spin_unlock_irqrestore(hba->host->host_lock, flags);
2045 }
2046 EXPORT_SYMBOL_GPL(ufshcd_release);
2047
ufshcd_clkgate_delay_show(struct device * dev,struct device_attribute * attr,char * buf)2048 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
2049 struct device_attribute *attr, char *buf)
2050 {
2051 struct ufs_hba *hba = dev_get_drvdata(dev);
2052
2053 return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
2054 }
2055
ufshcd_clkgate_delay_set(struct device * dev,unsigned long value)2056 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value)
2057 {
2058 struct ufs_hba *hba = dev_get_drvdata(dev);
2059 unsigned long flags;
2060
2061 spin_lock_irqsave(hba->host->host_lock, flags);
2062 hba->clk_gating.delay_ms = value;
2063 spin_unlock_irqrestore(hba->host->host_lock, flags);
2064 }
2065 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set);
2066
ufshcd_clkgate_delay_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2067 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
2068 struct device_attribute *attr, const char *buf, size_t count)
2069 {
2070 unsigned long value;
2071
2072 if (kstrtoul(buf, 0, &value))
2073 return -EINVAL;
2074
2075 ufshcd_clkgate_delay_set(dev, value);
2076 return count;
2077 }
2078
ufshcd_clkgate_enable_show(struct device * dev,struct device_attribute * attr,char * buf)2079 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
2080 struct device_attribute *attr, char *buf)
2081 {
2082 struct ufs_hba *hba = dev_get_drvdata(dev);
2083
2084 return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
2085 }
2086
ufshcd_clkgate_enable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2087 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
2088 struct device_attribute *attr, const char *buf, size_t count)
2089 {
2090 struct ufs_hba *hba = dev_get_drvdata(dev);
2091 unsigned long flags;
2092 u32 value;
2093
2094 if (kstrtou32(buf, 0, &value))
2095 return -EINVAL;
2096
2097 value = !!value;
2098
2099 spin_lock_irqsave(hba->host->host_lock, flags);
2100 if (value == hba->clk_gating.is_enabled)
2101 goto out;
2102
2103 if (value)
2104 __ufshcd_release(hba);
2105 else
2106 hba->clk_gating.active_reqs++;
2107
2108 hba->clk_gating.is_enabled = value;
2109 out:
2110 spin_unlock_irqrestore(hba->host->host_lock, flags);
2111 return count;
2112 }
2113
ufshcd_init_clk_gating_sysfs(struct ufs_hba * hba)2114 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
2115 {
2116 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
2117 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
2118 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
2119 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
2120 hba->clk_gating.delay_attr.attr.mode = 0644;
2121 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
2122 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
2123
2124 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
2125 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
2126 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
2127 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
2128 hba->clk_gating.enable_attr.attr.mode = 0644;
2129 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
2130 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
2131 }
2132
ufshcd_remove_clk_gating_sysfs(struct ufs_hba * hba)2133 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
2134 {
2135 if (hba->clk_gating.delay_attr.attr.name)
2136 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
2137 if (hba->clk_gating.enable_attr.attr.name)
2138 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
2139 }
2140
ufshcd_init_clk_gating(struct ufs_hba * hba)2141 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
2142 {
2143 if (!ufshcd_is_clkgating_allowed(hba))
2144 return;
2145
2146 hba->clk_gating.state = CLKS_ON;
2147
2148 hba->clk_gating.delay_ms = 150;
2149 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
2150 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
2151
2152 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(
2153 "ufs_clk_gating_%d", WQ_MEM_RECLAIM | WQ_HIGHPRI,
2154 hba->host->host_no);
2155
2156 ufshcd_init_clk_gating_sysfs(hba);
2157
2158 hba->clk_gating.is_enabled = true;
2159 hba->clk_gating.is_initialized = true;
2160 }
2161
ufshcd_exit_clk_gating(struct ufs_hba * hba)2162 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
2163 {
2164 if (!hba->clk_gating.is_initialized)
2165 return;
2166
2167 ufshcd_remove_clk_gating_sysfs(hba);
2168
2169 /* Ungate the clock if necessary. */
2170 ufshcd_hold(hba);
2171 hba->clk_gating.is_initialized = false;
2172 ufshcd_release(hba);
2173
2174 destroy_workqueue(hba->clk_gating.clk_gating_workq);
2175 }
2176
ufshcd_clk_scaling_start_busy(struct ufs_hba * hba)2177 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
2178 {
2179 bool queue_resume_work = false;
2180 ktime_t curr_t = ktime_get();
2181 unsigned long flags;
2182
2183 if (!ufshcd_is_clkscaling_supported(hba))
2184 return;
2185
2186 spin_lock_irqsave(hba->host->host_lock, flags);
2187 if (!hba->clk_scaling.active_reqs++)
2188 queue_resume_work = true;
2189
2190 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
2191 spin_unlock_irqrestore(hba->host->host_lock, flags);
2192 return;
2193 }
2194
2195 if (queue_resume_work)
2196 queue_work(hba->clk_scaling.workq,
2197 &hba->clk_scaling.resume_work);
2198
2199 if (!hba->clk_scaling.window_start_t) {
2200 hba->clk_scaling.window_start_t = curr_t;
2201 hba->clk_scaling.tot_busy_t = 0;
2202 hba->clk_scaling.is_busy_started = false;
2203 }
2204
2205 if (!hba->clk_scaling.is_busy_started) {
2206 hba->clk_scaling.busy_start_t = curr_t;
2207 hba->clk_scaling.is_busy_started = true;
2208 }
2209 spin_unlock_irqrestore(hba->host->host_lock, flags);
2210 }
2211
ufshcd_clk_scaling_update_busy(struct ufs_hba * hba)2212 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2213 {
2214 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2215 unsigned long flags;
2216
2217 if (!ufshcd_is_clkscaling_supported(hba))
2218 return;
2219
2220 spin_lock_irqsave(hba->host->host_lock, flags);
2221 hba->clk_scaling.active_reqs--;
2222 if (!scaling->active_reqs && scaling->is_busy_started) {
2223 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2224 scaling->busy_start_t));
2225 scaling->busy_start_t = 0;
2226 scaling->is_busy_started = false;
2227 }
2228 spin_unlock_irqrestore(hba->host->host_lock, flags);
2229 }
2230
ufshcd_monitor_opcode2dir(u8 opcode)2231 static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2232 {
2233 if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2234 return READ;
2235 else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2236 return WRITE;
2237 else
2238 return -EINVAL;
2239 }
2240
ufshcd_should_inform_monitor(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2241 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2242 struct ufshcd_lrb *lrbp)
2243 {
2244 const struct ufs_hba_monitor *m = &hba->monitor;
2245
2246 return (m->enabled && lrbp && lrbp->cmd &&
2247 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2248 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2249 }
2250
ufshcd_start_monitor(struct ufs_hba * hba,const struct ufshcd_lrb * lrbp)2251 static void ufshcd_start_monitor(struct ufs_hba *hba,
2252 const struct ufshcd_lrb *lrbp)
2253 {
2254 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2255 unsigned long flags;
2256
2257 spin_lock_irqsave(hba->host->host_lock, flags);
2258 if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2259 hba->monitor.busy_start_ts[dir] = ktime_get();
2260 spin_unlock_irqrestore(hba->host->host_lock, flags);
2261 }
2262
ufshcd_update_monitor(struct ufs_hba * hba,const struct ufshcd_lrb * lrbp)2263 static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp)
2264 {
2265 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2266 unsigned long flags;
2267
2268 spin_lock_irqsave(hba->host->host_lock, flags);
2269 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2270 const struct request *req = scsi_cmd_to_rq(lrbp->cmd);
2271 struct ufs_hba_monitor *m = &hba->monitor;
2272 ktime_t now, inc, lat;
2273
2274 now = lrbp->compl_time_stamp;
2275 inc = ktime_sub(now, m->busy_start_ts[dir]);
2276 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2277 m->nr_sec_rw[dir] += blk_rq_sectors(req);
2278
2279 /* Update latencies */
2280 m->nr_req[dir]++;
2281 lat = ktime_sub(now, lrbp->issue_time_stamp);
2282 m->lat_sum[dir] += lat;
2283 if (m->lat_max[dir] < lat || !m->lat_max[dir])
2284 m->lat_max[dir] = lat;
2285 if (m->lat_min[dir] > lat || !m->lat_min[dir])
2286 m->lat_min[dir] = lat;
2287
2288 m->nr_queued[dir]--;
2289 /* Push forward the busy start of monitor */
2290 m->busy_start_ts[dir] = now;
2291 }
2292 spin_unlock_irqrestore(hba->host->host_lock, flags);
2293 }
2294
2295 /**
2296 * ufshcd_send_command - Send SCSI or device management commands
2297 * @hba: per adapter instance
2298 * @task_tag: Task tag of the command
2299 * @hwq: pointer to hardware queue instance
2300 */
2301 static inline
ufshcd_send_command(struct ufs_hba * hba,unsigned int task_tag,struct ufs_hw_queue * hwq)2302 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag,
2303 struct ufs_hw_queue *hwq)
2304 {
2305 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2306 unsigned long flags;
2307
2308 lrbp->issue_time_stamp = ktime_get();
2309 lrbp->issue_time_stamp_local_clock = local_clock();
2310 lrbp->compl_time_stamp = ktime_set(0, 0);
2311 lrbp->compl_time_stamp_local_clock = 0;
2312 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
2313 if (lrbp->cmd)
2314 ufshcd_clk_scaling_start_busy(hba);
2315 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2316 ufshcd_start_monitor(hba, lrbp);
2317
2318 if (hba->mcq_enabled) {
2319 int utrd_size = sizeof(struct utp_transfer_req_desc);
2320 struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr;
2321 struct utp_transfer_req_desc *dest;
2322
2323 spin_lock(&hwq->sq_lock);
2324 dest = hwq->sqe_base_addr + hwq->sq_tail_slot;
2325 memcpy(dest, src, utrd_size);
2326 ufshcd_inc_sq_tail(hwq);
2327 spin_unlock(&hwq->sq_lock);
2328 } else {
2329 spin_lock_irqsave(&hba->outstanding_lock, flags);
2330 if (hba->vops && hba->vops->setup_xfer_req)
2331 hba->vops->setup_xfer_req(hba, lrbp->task_tag,
2332 !!lrbp->cmd);
2333 __set_bit(lrbp->task_tag, &hba->outstanding_reqs);
2334 ufshcd_writel(hba, 1 << lrbp->task_tag,
2335 REG_UTP_TRANSFER_REQ_DOOR_BELL);
2336 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2337 }
2338 }
2339
2340 /**
2341 * ufshcd_copy_sense_data - Copy sense data in case of check condition
2342 * @lrbp: pointer to local reference block
2343 */
ufshcd_copy_sense_data(struct ufshcd_lrb * lrbp)2344 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2345 {
2346 u8 *const sense_buffer = lrbp->cmd->sense_buffer;
2347 u16 resp_len;
2348 int len;
2349
2350 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length);
2351 if (sense_buffer && resp_len) {
2352 int len_to_copy;
2353
2354 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2355 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
2356
2357 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2358 len_to_copy);
2359 }
2360 }
2361
2362 /**
2363 * ufshcd_copy_query_response() - Copy the Query Response and the data
2364 * descriptor
2365 * @hba: per adapter instance
2366 * @lrbp: pointer to local reference block
2367 *
2368 * Return: 0 upon success; < 0 upon failure.
2369 */
2370 static
ufshcd_copy_query_response(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2371 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2372 {
2373 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2374
2375 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2376
2377 /* Get the descriptor */
2378 if (hba->dev_cmd.query.descriptor &&
2379 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2380 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2381 GENERAL_UPIU_REQUEST_SIZE;
2382 u16 resp_len;
2383 u16 buf_len;
2384
2385 /* data segment length */
2386 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
2387 .data_segment_length);
2388 buf_len = be16_to_cpu(
2389 hba->dev_cmd.query.request.upiu_req.length);
2390 if (likely(buf_len >= resp_len)) {
2391 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2392 } else {
2393 dev_warn(hba->dev,
2394 "%s: rsp size %d is bigger than buffer size %d",
2395 __func__, resp_len, buf_len);
2396 return -EINVAL;
2397 }
2398 }
2399
2400 return 0;
2401 }
2402
2403 /**
2404 * ufshcd_hba_capabilities - Read controller capabilities
2405 * @hba: per adapter instance
2406 *
2407 * Return: 0 on success, negative on error.
2408 */
ufshcd_hba_capabilities(struct ufs_hba * hba)2409 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2410 {
2411 int err;
2412
2413 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2414 if (hba->quirks & UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS)
2415 hba->capabilities &= ~MASK_64_ADDRESSING_SUPPORT;
2416
2417 /* nutrs and nutmrs are 0 based values */
2418 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1;
2419 hba->nutmrs =
2420 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2421 hba->reserved_slot = hba->nutrs - 1;
2422
2423 hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1;
2424
2425 /* Read crypto capabilities */
2426 err = ufshcd_hba_init_crypto_capabilities(hba);
2427 if (err) {
2428 dev_err(hba->dev, "crypto setup failed\n");
2429 return err;
2430 }
2431
2432 /*
2433 * The UFSHCI 3.0 specification does not define MCQ_SUPPORT and
2434 * LSDB_SUPPORT, but [31:29] as reserved bits with reset value 0s, which
2435 * means we can simply read values regardless of version.
2436 */
2437 hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities);
2438 /*
2439 * 0h: legacy single doorbell support is available
2440 * 1h: indicate that legacy single doorbell support has been removed
2441 */
2442 if (!(hba->quirks & UFSHCD_QUIRK_BROKEN_LSDBS_CAP))
2443 hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities);
2444 else
2445 hba->lsdb_sup = true;
2446
2447 if (!hba->mcq_sup)
2448 return 0;
2449
2450 hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP);
2451 hba->ext_iid_sup = FIELD_GET(MASK_EXT_IID_SUPPORT,
2452 hba->mcq_capabilities);
2453
2454 return 0;
2455 }
2456
2457 /**
2458 * ufshcd_ready_for_uic_cmd - Check if controller is ready
2459 * to accept UIC commands
2460 * @hba: per adapter instance
2461 *
2462 * Return: true on success, else false.
2463 */
ufshcd_ready_for_uic_cmd(struct ufs_hba * hba)2464 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2465 {
2466 u32 val;
2467 int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY,
2468 500, uic_cmd_timeout * 1000, false, hba,
2469 REG_CONTROLLER_STATUS);
2470 return ret == 0;
2471 }
2472
2473 /**
2474 * ufshcd_get_upmcrs - Get the power mode change request status
2475 * @hba: Pointer to adapter instance
2476 *
2477 * This function gets the UPMCRS field of HCS register
2478 *
2479 * Return: value of UPMCRS field.
2480 */
ufshcd_get_upmcrs(struct ufs_hba * hba)2481 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2482 {
2483 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2484 }
2485
2486 /**
2487 * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer
2488 * @hba: per adapter instance
2489 * @uic_cmd: UIC command
2490 */
2491 static inline void
ufshcd_dispatch_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2492 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2493 {
2494 lockdep_assert_held(&hba->uic_cmd_mutex);
2495
2496 WARN_ON(hba->active_uic_cmd);
2497
2498 hba->active_uic_cmd = uic_cmd;
2499
2500 /* Write Args */
2501 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2502 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2503 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2504
2505 ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
2506
2507 /* Write UIC Cmd */
2508 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2509 REG_UIC_COMMAND);
2510 }
2511
2512 /**
2513 * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command
2514 * @hba: per adapter instance
2515 * @uic_cmd: UIC command
2516 *
2517 * Return: 0 only if success.
2518 */
2519 static int
ufshcd_wait_for_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2520 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2521 {
2522 int ret;
2523 unsigned long flags;
2524
2525 lockdep_assert_held(&hba->uic_cmd_mutex);
2526
2527 if (wait_for_completion_timeout(&uic_cmd->done,
2528 msecs_to_jiffies(uic_cmd_timeout))) {
2529 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2530 } else {
2531 ret = -ETIMEDOUT;
2532 dev_err(hba->dev,
2533 "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2534 uic_cmd->command, uic_cmd->argument3);
2535
2536 if (!uic_cmd->cmd_active) {
2537 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2538 __func__);
2539 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2540 }
2541 }
2542
2543 spin_lock_irqsave(hba->host->host_lock, flags);
2544 hba->active_uic_cmd = NULL;
2545 spin_unlock_irqrestore(hba->host->host_lock, flags);
2546
2547 return ret;
2548 }
2549
2550 /**
2551 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2552 * @hba: per adapter instance
2553 * @uic_cmd: UIC command
2554 * @completion: initialize the completion only if this is set to true
2555 *
2556 * Return: 0 only if success.
2557 */
2558 static int
__ufshcd_send_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd,bool completion)2559 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2560 bool completion)
2561 {
2562 lockdep_assert_held(&hba->uic_cmd_mutex);
2563
2564 if (!ufshcd_ready_for_uic_cmd(hba)) {
2565 dev_err(hba->dev,
2566 "Controller not ready to accept UIC commands\n");
2567 return -EIO;
2568 }
2569
2570 if (completion)
2571 init_completion(&uic_cmd->done);
2572
2573 uic_cmd->cmd_active = 1;
2574 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2575
2576 return 0;
2577 }
2578
2579 /**
2580 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2581 * @hba: per adapter instance
2582 * @uic_cmd: UIC command
2583 *
2584 * Return: 0 only if success.
2585 */
ufshcd_send_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2586 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2587 {
2588 int ret;
2589
2590 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
2591 return 0;
2592
2593 ufshcd_hold(hba);
2594 mutex_lock(&hba->uic_cmd_mutex);
2595 ufshcd_add_delay_before_dme_cmd(hba);
2596
2597 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2598 if (!ret)
2599 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2600
2601 mutex_unlock(&hba->uic_cmd_mutex);
2602
2603 ufshcd_release(hba);
2604 return ret;
2605 }
2606
2607 /**
2608 * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format)
2609 * @hba: per-adapter instance
2610 * @lrbp: pointer to local reference block
2611 * @sg_entries: The number of sg lists actually used
2612 * @sg_list: Pointer to SG list
2613 */
ufshcd_sgl_to_prdt(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,int sg_entries,struct scatterlist * sg_list)2614 static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries,
2615 struct scatterlist *sg_list)
2616 {
2617 struct ufshcd_sg_entry *prd;
2618 struct scatterlist *sg;
2619 int i;
2620
2621 if (sg_entries) {
2622
2623 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2624 lrbp->utr_descriptor_ptr->prd_table_length =
2625 cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba));
2626 else
2627 lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries);
2628
2629 prd = lrbp->ucd_prdt_ptr;
2630
2631 for_each_sg(sg_list, sg, sg_entries, i) {
2632 const unsigned int len = sg_dma_len(sg);
2633
2634 /*
2635 * From the UFSHCI spec: "Data Byte Count (DBC): A '0'
2636 * based value that indicates the length, in bytes, of
2637 * the data block. A maximum of length of 256KB may
2638 * exist for any entry. Bits 1:0 of this field shall be
2639 * 11b to indicate Dword granularity. A value of '3'
2640 * indicates 4 bytes, '7' indicates 8 bytes, etc."
2641 */
2642 WARN_ONCE(len > SZ_256K, "len = %#x\n", len);
2643 prd->size = cpu_to_le32(len - 1);
2644 prd->addr = cpu_to_le64(sg->dma_address);
2645 prd->reserved = 0;
2646 prd = (void *)prd + ufshcd_sg_entry_size(hba);
2647 }
2648 } else {
2649 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2650 }
2651 }
2652
2653 /**
2654 * ufshcd_map_sg - Map scatter-gather list to prdt
2655 * @hba: per adapter instance
2656 * @lrbp: pointer to local reference block
2657 *
2658 * Return: 0 in case of success, non-zero value in case of failure.
2659 */
ufshcd_map_sg(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2660 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2661 {
2662 struct scsi_cmnd *cmd = lrbp->cmd;
2663 int sg_segments = scsi_dma_map(cmd);
2664
2665 if (sg_segments < 0)
2666 return sg_segments;
2667
2668 ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));
2669
2670 return ufshcd_crypto_fill_prdt(hba, lrbp);
2671 }
2672
2673 /**
2674 * ufshcd_enable_intr - enable interrupts
2675 * @hba: per adapter instance
2676 * @intrs: interrupt bits
2677 */
ufshcd_enable_intr(struct ufs_hba * hba,u32 intrs)2678 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2679 {
2680 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2681
2682 set |= intrs;
2683 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2684 }
2685
2686 /**
2687 * ufshcd_disable_intr - disable interrupts
2688 * @hba: per adapter instance
2689 * @intrs: interrupt bits
2690 */
ufshcd_disable_intr(struct ufs_hba * hba,u32 intrs)2691 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2692 {
2693 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2694
2695 set &= ~intrs;
2696 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2697 }
2698
2699 /**
2700 * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request
2701 * descriptor according to request
2702 * @hba: per adapter instance
2703 * @lrbp: pointer to local reference block
2704 * @upiu_flags: flags required in the header
2705 * @cmd_dir: requests data direction
2706 * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments)
2707 */
2708 static void
ufshcd_prepare_req_desc_hdr(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,u8 * upiu_flags,enum dma_data_direction cmd_dir,int ehs_length)2709 ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
2710 u8 *upiu_flags, enum dma_data_direction cmd_dir,
2711 int ehs_length)
2712 {
2713 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2714 struct request_desc_header *h = &req_desc->header;
2715 enum utp_data_direction data_direction;
2716
2717 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2718
2719 *h = (typeof(*h)){ };
2720
2721 if (cmd_dir == DMA_FROM_DEVICE) {
2722 data_direction = UTP_DEVICE_TO_HOST;
2723 *upiu_flags = UPIU_CMD_FLAGS_READ;
2724 } else if (cmd_dir == DMA_TO_DEVICE) {
2725 data_direction = UTP_HOST_TO_DEVICE;
2726 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2727 } else {
2728 data_direction = UTP_NO_DATA_TRANSFER;
2729 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2730 }
2731
2732 h->command_type = lrbp->command_type;
2733 h->data_direction = data_direction;
2734 h->ehs_length = ehs_length;
2735
2736 if (lrbp->intr_cmd)
2737 h->interrupt = 1;
2738
2739 /* Prepare crypto related dwords */
2740 ufshcd_prepare_req_desc_hdr_crypto(lrbp, h);
2741
2742 /*
2743 * assigning invalid value for command status. Controller
2744 * updates OCS on command completion, with the command
2745 * status
2746 */
2747 h->ocs = OCS_INVALID_COMMAND_STATUS;
2748
2749 req_desc->prd_table_length = 0;
2750 }
2751
2752 /**
2753 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2754 * for scsi commands
2755 * @lrbp: local reference block pointer
2756 * @upiu_flags: flags
2757 */
2758 static
ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb * lrbp,u8 upiu_flags)2759 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2760 {
2761 struct scsi_cmnd *cmd = lrbp->cmd;
2762 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2763 unsigned short cdb_len;
2764
2765 ucd_req_ptr->header = (struct utp_upiu_header){
2766 .transaction_code = UPIU_TRANSACTION_COMMAND,
2767 .flags = upiu_flags,
2768 .lun = lrbp->lun,
2769 .task_tag = lrbp->task_tag,
2770 .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI,
2771 };
2772
2773 WARN_ON_ONCE(ucd_req_ptr->header.task_tag != lrbp->task_tag);
2774
2775 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2776
2777 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2778 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2779 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2780
2781 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2782 }
2783
2784 /**
2785 * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request
2786 * @hba: UFS hba
2787 * @lrbp: local reference block pointer
2788 * @upiu_flags: flags
2789 */
ufshcd_prepare_utp_query_req_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,u8 upiu_flags)2790 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2791 struct ufshcd_lrb *lrbp, u8 upiu_flags)
2792 {
2793 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2794 struct ufs_query *query = &hba->dev_cmd.query;
2795 u16 len = be16_to_cpu(query->request.upiu_req.length);
2796
2797 /* Query request header */
2798 ucd_req_ptr->header = (struct utp_upiu_header){
2799 .transaction_code = UPIU_TRANSACTION_QUERY_REQ,
2800 .flags = upiu_flags,
2801 .lun = lrbp->lun,
2802 .task_tag = lrbp->task_tag,
2803 .query_function = query->request.query_func,
2804 /* Data segment length only need for WRITE_DESC */
2805 .data_segment_length =
2806 query->request.upiu_req.opcode ==
2807 UPIU_QUERY_OPCODE_WRITE_DESC ?
2808 cpu_to_be16(len) :
2809 0,
2810 };
2811
2812 /* Copy the Query Request buffer as is */
2813 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2814 QUERY_OSF_SIZE);
2815
2816 /* Copy the Descriptor */
2817 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2818 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2819
2820 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2821 }
2822
ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb * lrbp)2823 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2824 {
2825 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2826
2827 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2828
2829 ucd_req_ptr->header = (struct utp_upiu_header){
2830 .transaction_code = UPIU_TRANSACTION_NOP_OUT,
2831 .task_tag = lrbp->task_tag,
2832 };
2833
2834 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2835 }
2836
2837 /**
2838 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2839 * for Device Management Purposes
2840 * @hba: per adapter instance
2841 * @lrbp: pointer to local reference block
2842 *
2843 * Return: 0 upon success; < 0 upon failure.
2844 */
ufshcd_compose_devman_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2845 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2846 struct ufshcd_lrb *lrbp)
2847 {
2848 u8 upiu_flags;
2849 int ret = 0;
2850
2851 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
2852
2853 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2854 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2855 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2856 ufshcd_prepare_utp_nop_upiu(lrbp);
2857 else
2858 ret = -EINVAL;
2859
2860 return ret;
2861 }
2862
2863 /**
2864 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2865 * for SCSI Purposes
2866 * @hba: per adapter instance
2867 * @lrbp: pointer to local reference block
2868 */
ufshcd_comp_scsi_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2869 static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2870 {
2871 struct request *rq = scsi_cmd_to_rq(lrbp->cmd);
2872 unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
2873 u8 upiu_flags;
2874
2875 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0);
2876 if (ioprio_class == IOPRIO_CLASS_RT)
2877 upiu_flags |= UPIU_CMD_FLAGS_CP;
2878 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2879 }
2880
2881 /**
2882 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2883 * @upiu_wlun_id: UPIU W-LUN id
2884 *
2885 * Return: SCSI W-LUN id.
2886 */
ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)2887 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2888 {
2889 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2890 }
2891
is_device_wlun(struct scsi_device * sdev)2892 static inline bool is_device_wlun(struct scsi_device *sdev)
2893 {
2894 return sdev->lun ==
2895 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN);
2896 }
2897
2898 /*
2899 * Associate the UFS controller queue with the default and poll HCTX types.
2900 * Initialize the mq_map[] arrays.
2901 */
ufshcd_map_queues(struct Scsi_Host * shost)2902 static void ufshcd_map_queues(struct Scsi_Host *shost)
2903 {
2904 struct ufs_hba *hba = shost_priv(shost);
2905 int i, queue_offset = 0;
2906
2907 if (!is_mcq_supported(hba)) {
2908 hba->nr_queues[HCTX_TYPE_DEFAULT] = 1;
2909 hba->nr_queues[HCTX_TYPE_READ] = 0;
2910 hba->nr_queues[HCTX_TYPE_POLL] = 1;
2911 hba->nr_hw_queues = 1;
2912 }
2913
2914 for (i = 0; i < shost->nr_maps; i++) {
2915 struct blk_mq_queue_map *map = &shost->tag_set.map[i];
2916
2917 map->nr_queues = hba->nr_queues[i];
2918 if (!map->nr_queues)
2919 continue;
2920 map->queue_offset = queue_offset;
2921 if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba))
2922 map->queue_offset = 0;
2923
2924 blk_mq_map_queues(map);
2925 queue_offset += map->nr_queues;
2926 }
2927 }
2928
ufshcd_init_lrb(struct ufs_hba * hba,struct ufshcd_lrb * lrb,int i)2929 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2930 {
2931 struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr +
2932 i * ufshcd_get_ucd_size(hba);
2933 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2934 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2935 i * ufshcd_get_ucd_size(hba);
2936 u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset);
2937 u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset);
2938
2939 lrb->utr_descriptor_ptr = utrdlp + i;
2940 lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2941 i * sizeof(struct utp_transfer_req_desc);
2942 lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu;
2943 lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2944 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu;
2945 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2946 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table;
2947 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2948 }
2949
2950 /**
2951 * ufshcd_queuecommand - main entry point for SCSI requests
2952 * @host: SCSI host pointer
2953 * @cmd: command from SCSI Midlayer
2954 *
2955 * Return: 0 for success, non-zero in case of failure.
2956 */
ufshcd_queuecommand(struct Scsi_Host * host,struct scsi_cmnd * cmd)2957 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2958 {
2959 struct ufs_hba *hba = shost_priv(host);
2960 int tag = scsi_cmd_to_rq(cmd)->tag;
2961 struct ufshcd_lrb *lrbp;
2962 int err = 0;
2963 struct ufs_hw_queue *hwq = NULL;
2964
2965 switch (hba->ufshcd_state) {
2966 case UFSHCD_STATE_OPERATIONAL:
2967 break;
2968 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2969 /*
2970 * SCSI error handler can call ->queuecommand() while UFS error
2971 * handler is in progress. Error interrupts could change the
2972 * state from UFSHCD_STATE_RESET to
2973 * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests
2974 * being issued in that case.
2975 */
2976 if (ufshcd_eh_in_progress(hba)) {
2977 err = SCSI_MLQUEUE_HOST_BUSY;
2978 goto out;
2979 }
2980 break;
2981 case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2982 /*
2983 * pm_runtime_get_sync() is used at error handling preparation
2984 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2985 * PM ops, it can never be finished if we let SCSI layer keep
2986 * retrying it, which gets err handler stuck forever. Neither
2987 * can we let the scsi cmd pass through, because UFS is in bad
2988 * state, the scsi cmd may eventually time out, which will get
2989 * err handler blocked for too long. So, just fail the scsi cmd
2990 * sent from PM ops, err handler can recover PM error anyways.
2991 */
2992 if (hba->pm_op_in_progress) {
2993 hba->force_reset = true;
2994 set_host_byte(cmd, DID_BAD_TARGET);
2995 scsi_done(cmd);
2996 goto out;
2997 }
2998 fallthrough;
2999 case UFSHCD_STATE_RESET:
3000 err = SCSI_MLQUEUE_HOST_BUSY;
3001 goto out;
3002 case UFSHCD_STATE_ERROR:
3003 set_host_byte(cmd, DID_ERROR);
3004 scsi_done(cmd);
3005 goto out;
3006 }
3007
3008 hba->req_abort_count = 0;
3009
3010 ufshcd_hold(hba);
3011
3012 lrbp = &hba->lrb[tag];
3013 lrbp->cmd = cmd;
3014 lrbp->task_tag = tag;
3015 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
3016 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
3017
3018 ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp);
3019
3020 lrbp->req_abort_skip = false;
3021
3022 ufshcd_comp_scsi_upiu(hba, lrbp);
3023
3024 err = ufshcd_map_sg(hba, lrbp);
3025 if (err) {
3026 ufshcd_release(hba);
3027 goto out;
3028 }
3029
3030 if (hba->mcq_enabled)
3031 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
3032
3033 ufshcd_send_command(hba, tag, hwq);
3034
3035 out:
3036 if (ufs_trigger_eh(hba)) {
3037 unsigned long flags;
3038
3039 spin_lock_irqsave(hba->host->host_lock, flags);
3040 ufshcd_schedule_eh_work(hba);
3041 spin_unlock_irqrestore(hba->host->host_lock, flags);
3042 }
3043
3044 return err;
3045 }
3046
ufshcd_setup_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,enum dev_cmd_type cmd_type,u8 lun,int tag)3047 static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
3048 enum dev_cmd_type cmd_type, u8 lun, int tag)
3049 {
3050 lrbp->cmd = NULL;
3051 lrbp->task_tag = tag;
3052 lrbp->lun = lun;
3053 lrbp->intr_cmd = true; /* No interrupt aggregation */
3054 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
3055 hba->dev_cmd.type = cmd_type;
3056 }
3057
ufshcd_compose_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,enum dev_cmd_type cmd_type,int tag)3058 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
3059 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
3060 {
3061 ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
3062
3063 return ufshcd_compose_devman_upiu(hba, lrbp);
3064 }
3065
3066 /*
3067 * Check with the block layer if the command is inflight
3068 * @cmd: command to check.
3069 *
3070 * Return: true if command is inflight; false if not.
3071 */
ufshcd_cmd_inflight(struct scsi_cmnd * cmd)3072 bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd)
3073 {
3074 return cmd && blk_mq_rq_state(scsi_cmd_to_rq(cmd)) == MQ_RQ_IN_FLIGHT;
3075 }
3076
3077 /*
3078 * Clear the pending command in the controller and wait until
3079 * the controller confirms that the command has been cleared.
3080 * @hba: per adapter instance
3081 * @task_tag: The tag number of the command to be cleared.
3082 */
ufshcd_clear_cmd(struct ufs_hba * hba,u32 task_tag)3083 static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag)
3084 {
3085 u32 mask;
3086 unsigned long flags;
3087 int err;
3088
3089 if (hba->mcq_enabled) {
3090 /*
3091 * MCQ mode. Clean up the MCQ resources similar to
3092 * what the ufshcd_utrl_clear() does for SDB mode.
3093 */
3094 err = ufshcd_mcq_sq_cleanup(hba, task_tag);
3095 if (err) {
3096 dev_err(hba->dev, "%s: failed tag=%d. err=%d\n",
3097 __func__, task_tag, err);
3098 return err;
3099 }
3100 return 0;
3101 }
3102
3103 mask = 1U << task_tag;
3104
3105 /* clear outstanding transaction before retry */
3106 spin_lock_irqsave(hba->host->host_lock, flags);
3107 ufshcd_utrl_clear(hba, mask);
3108 spin_unlock_irqrestore(hba->host->host_lock, flags);
3109
3110 /*
3111 * wait for h/w to clear corresponding bit in door-bell.
3112 * max. wait is 1 sec.
3113 */
3114 return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL,
3115 mask, ~mask, 1000, 1000);
3116 }
3117
3118 /**
3119 * ufshcd_dev_cmd_completion() - handles device management command responses
3120 * @hba: per adapter instance
3121 * @lrbp: pointer to local reference block
3122 *
3123 * Return: 0 upon success; < 0 upon failure.
3124 */
3125 static int
ufshcd_dev_cmd_completion(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)3126 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
3127 {
3128 enum upiu_response_transaction resp;
3129 int err = 0;
3130
3131 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
3132 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
3133
3134 switch (resp) {
3135 case UPIU_TRANSACTION_NOP_IN:
3136 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
3137 err = -EINVAL;
3138 dev_err(hba->dev, "%s: unexpected response %x\n",
3139 __func__, resp);
3140 }
3141 break;
3142 case UPIU_TRANSACTION_QUERY_RSP: {
3143 u8 response = lrbp->ucd_rsp_ptr->header.response;
3144
3145 if (response == 0)
3146 err = ufshcd_copy_query_response(hba, lrbp);
3147 break;
3148 }
3149 case UPIU_TRANSACTION_REJECT_UPIU:
3150 /* TODO: handle Reject UPIU Response */
3151 err = -EPERM;
3152 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
3153 __func__);
3154 break;
3155 case UPIU_TRANSACTION_RESPONSE:
3156 if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) {
3157 err = -EINVAL;
3158 dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp);
3159 }
3160 break;
3161 default:
3162 err = -EINVAL;
3163 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
3164 __func__, resp);
3165 break;
3166 }
3167
3168 return err;
3169 }
3170
ufshcd_wait_for_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,int max_timeout)3171 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
3172 struct ufshcd_lrb *lrbp, int max_timeout)
3173 {
3174 unsigned long time_left = msecs_to_jiffies(max_timeout);
3175 unsigned long flags;
3176 bool pending;
3177 int err;
3178
3179 retry:
3180 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
3181 time_left);
3182
3183 if (likely(time_left)) {
3184 /*
3185 * The completion handler called complete() and the caller of
3186 * this function still owns the @lrbp tag so the code below does
3187 * not trigger any race conditions.
3188 */
3189 hba->dev_cmd.complete = NULL;
3190 err = ufshcd_get_tr_ocs(lrbp, NULL);
3191 if (!err)
3192 err = ufshcd_dev_cmd_completion(hba, lrbp);
3193 } else {
3194 err = -ETIMEDOUT;
3195 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
3196 __func__, lrbp->task_tag);
3197
3198 /* MCQ mode */
3199 if (hba->mcq_enabled) {
3200 /* successfully cleared the command, retry if needed */
3201 if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0)
3202 err = -EAGAIN;
3203 hba->dev_cmd.complete = NULL;
3204 return err;
3205 }
3206
3207 /* SDB mode */
3208 if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) {
3209 /* successfully cleared the command, retry if needed */
3210 err = -EAGAIN;
3211 /*
3212 * Since clearing the command succeeded we also need to
3213 * clear the task tag bit from the outstanding_reqs
3214 * variable.
3215 */
3216 spin_lock_irqsave(&hba->outstanding_lock, flags);
3217 pending = test_bit(lrbp->task_tag,
3218 &hba->outstanding_reqs);
3219 if (pending) {
3220 hba->dev_cmd.complete = NULL;
3221 __clear_bit(lrbp->task_tag,
3222 &hba->outstanding_reqs);
3223 }
3224 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3225
3226 if (!pending) {
3227 /*
3228 * The completion handler ran while we tried to
3229 * clear the command.
3230 */
3231 time_left = 1;
3232 goto retry;
3233 }
3234 } else {
3235 dev_err(hba->dev, "%s: failed to clear tag %d\n",
3236 __func__, lrbp->task_tag);
3237
3238 spin_lock_irqsave(&hba->outstanding_lock, flags);
3239 pending = test_bit(lrbp->task_tag,
3240 &hba->outstanding_reqs);
3241 if (pending)
3242 hba->dev_cmd.complete = NULL;
3243 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3244
3245 if (!pending) {
3246 /*
3247 * The completion handler ran while we tried to
3248 * clear the command.
3249 */
3250 time_left = 1;
3251 goto retry;
3252 }
3253 }
3254 }
3255
3256 return err;
3257 }
3258
ufshcd_dev_man_lock(struct ufs_hba * hba)3259 static void ufshcd_dev_man_lock(struct ufs_hba *hba)
3260 {
3261 ufshcd_hold(hba);
3262 mutex_lock(&hba->dev_cmd.lock);
3263 down_read(&hba->clk_scaling_lock);
3264 }
3265
ufshcd_dev_man_unlock(struct ufs_hba * hba)3266 static void ufshcd_dev_man_unlock(struct ufs_hba *hba)
3267 {
3268 up_read(&hba->clk_scaling_lock);
3269 mutex_unlock(&hba->dev_cmd.lock);
3270 ufshcd_release(hba);
3271 }
3272
ufshcd_issue_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,const u32 tag,int timeout)3273 static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
3274 const u32 tag, int timeout)
3275 {
3276 DECLARE_COMPLETION_ONSTACK(wait);
3277 int err;
3278
3279 hba->dev_cmd.complete = &wait;
3280
3281 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
3282
3283 ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
3284 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
3285
3286 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
3287 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
3288
3289 return err;
3290 }
3291
3292 /**
3293 * ufshcd_exec_dev_cmd - API for sending device management requests
3294 * @hba: UFS hba
3295 * @cmd_type: specifies the type (NOP, Query...)
3296 * @timeout: timeout in milliseconds
3297 *
3298 * Return: 0 upon success; < 0 upon failure.
3299 *
3300 * NOTE: Since there is only one available tag for device management commands,
3301 * it is expected you hold the hba->dev_cmd.lock mutex.
3302 */
ufshcd_exec_dev_cmd(struct ufs_hba * hba,enum dev_cmd_type cmd_type,int timeout)3303 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
3304 enum dev_cmd_type cmd_type, int timeout)
3305 {
3306 const u32 tag = hba->reserved_slot;
3307 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
3308 int err;
3309
3310 /* Protects use of hba->reserved_slot. */
3311 lockdep_assert_held(&hba->dev_cmd.lock);
3312
3313 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
3314 if (unlikely(err))
3315 return err;
3316
3317 return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout);
3318 }
3319
3320 /**
3321 * ufshcd_init_query() - init the query response and request parameters
3322 * @hba: per-adapter instance
3323 * @request: address of the request pointer to be initialized
3324 * @response: address of the response pointer to be initialized
3325 * @opcode: operation to perform
3326 * @idn: flag idn to access
3327 * @index: LU number to access
3328 * @selector: query/flag/descriptor further identification
3329 */
ufshcd_init_query(struct ufs_hba * hba,struct ufs_query_req ** request,struct ufs_query_res ** response,enum query_opcode opcode,u8 idn,u8 index,u8 selector)3330 static inline void ufshcd_init_query(struct ufs_hba *hba,
3331 struct ufs_query_req **request, struct ufs_query_res **response,
3332 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
3333 {
3334 *request = &hba->dev_cmd.query.request;
3335 *response = &hba->dev_cmd.query.response;
3336 memset(*request, 0, sizeof(struct ufs_query_req));
3337 memset(*response, 0, sizeof(struct ufs_query_res));
3338 (*request)->upiu_req.opcode = opcode;
3339 (*request)->upiu_req.idn = idn;
3340 (*request)->upiu_req.index = index;
3341 (*request)->upiu_req.selector = selector;
3342 }
3343
ufshcd_query_flag_retry(struct ufs_hba * hba,enum query_opcode opcode,enum flag_idn idn,u8 index,bool * flag_res)3344 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
3345 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
3346 {
3347 int ret;
3348 int retries;
3349
3350 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
3351 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
3352 if (ret)
3353 dev_dbg(hba->dev,
3354 "%s: failed with error %d, retries %d\n",
3355 __func__, ret, retries);
3356 else
3357 break;
3358 }
3359
3360 if (ret)
3361 dev_err(hba->dev,
3362 "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n",
3363 __func__, opcode, idn, ret, retries);
3364 return ret;
3365 }
3366
3367 /**
3368 * ufshcd_query_flag() - API function for sending flag query requests
3369 * @hba: per-adapter instance
3370 * @opcode: flag query to perform
3371 * @idn: flag idn to access
3372 * @index: flag index to access
3373 * @flag_res: the flag value after the query request completes
3374 *
3375 * Return: 0 for success, non-zero in case of failure.
3376 */
ufshcd_query_flag(struct ufs_hba * hba,enum query_opcode opcode,enum flag_idn idn,u8 index,bool * flag_res)3377 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
3378 enum flag_idn idn, u8 index, bool *flag_res)
3379 {
3380 struct ufs_query_req *request = NULL;
3381 struct ufs_query_res *response = NULL;
3382 int err, selector = 0;
3383 int timeout = QUERY_REQ_TIMEOUT;
3384
3385 BUG_ON(!hba);
3386
3387 ufshcd_dev_man_lock(hba);
3388
3389 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3390 selector);
3391
3392 switch (opcode) {
3393 case UPIU_QUERY_OPCODE_SET_FLAG:
3394 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3395 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3396 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3397 break;
3398 case UPIU_QUERY_OPCODE_READ_FLAG:
3399 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3400 if (!flag_res) {
3401 /* No dummy reads */
3402 dev_err(hba->dev, "%s: Invalid argument for read request\n",
3403 __func__);
3404 err = -EINVAL;
3405 goto out_unlock;
3406 }
3407 break;
3408 default:
3409 dev_err(hba->dev,
3410 "%s: Expected query flag opcode but got = %d\n",
3411 __func__, opcode);
3412 err = -EINVAL;
3413 goto out_unlock;
3414 }
3415
3416 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
3417
3418 if (err) {
3419 dev_err(hba->dev,
3420 "%s: Sending flag query for idn %d failed, err = %d\n",
3421 __func__, idn, err);
3422 goto out_unlock;
3423 }
3424
3425 if (flag_res)
3426 *flag_res = (be32_to_cpu(response->upiu_res.value) &
3427 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3428
3429 out_unlock:
3430 ufshcd_dev_man_unlock(hba);
3431 return err;
3432 }
3433
3434 /**
3435 * ufshcd_query_attr - API function for sending attribute requests
3436 * @hba: per-adapter instance
3437 * @opcode: attribute opcode
3438 * @idn: attribute idn to access
3439 * @index: index field
3440 * @selector: selector field
3441 * @attr_val: the attribute value after the query request completes
3442 *
3443 * Return: 0 for success, non-zero in case of failure.
3444 */
ufshcd_query_attr(struct ufs_hba * hba,enum query_opcode opcode,enum attr_idn idn,u8 index,u8 selector,u32 * attr_val)3445 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3446 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
3447 {
3448 struct ufs_query_req *request = NULL;
3449 struct ufs_query_res *response = NULL;
3450 int err;
3451
3452 BUG_ON(!hba);
3453
3454 if (!attr_val) {
3455 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3456 __func__, opcode);
3457 return -EINVAL;
3458 }
3459
3460 ufshcd_dev_man_lock(hba);
3461
3462 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3463 selector);
3464
3465 switch (opcode) {
3466 case UPIU_QUERY_OPCODE_WRITE_ATTR:
3467 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3468 request->upiu_req.value = cpu_to_be32(*attr_val);
3469 break;
3470 case UPIU_QUERY_OPCODE_READ_ATTR:
3471 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3472 break;
3473 default:
3474 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3475 __func__, opcode);
3476 err = -EINVAL;
3477 goto out_unlock;
3478 }
3479
3480 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3481
3482 if (err) {
3483 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3484 __func__, opcode, idn, index, err);
3485 goto out_unlock;
3486 }
3487
3488 *attr_val = be32_to_cpu(response->upiu_res.value);
3489
3490 out_unlock:
3491 ufshcd_dev_man_unlock(hba);
3492 return err;
3493 }
3494
3495 /**
3496 * ufshcd_query_attr_retry() - API function for sending query
3497 * attribute with retries
3498 * @hba: per-adapter instance
3499 * @opcode: attribute opcode
3500 * @idn: attribute idn to access
3501 * @index: index field
3502 * @selector: selector field
3503 * @attr_val: the attribute value after the query request
3504 * completes
3505 *
3506 * Return: 0 for success, non-zero in case of failure.
3507 */
ufshcd_query_attr_retry(struct ufs_hba * hba,enum query_opcode opcode,enum attr_idn idn,u8 index,u8 selector,u32 * attr_val)3508 int ufshcd_query_attr_retry(struct ufs_hba *hba,
3509 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3510 u32 *attr_val)
3511 {
3512 int ret = 0;
3513 u32 retries;
3514
3515 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3516 ret = ufshcd_query_attr(hba, opcode, idn, index,
3517 selector, attr_val);
3518 if (ret)
3519 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3520 __func__, ret, retries);
3521 else
3522 break;
3523 }
3524
3525 if (ret)
3526 dev_err(hba->dev,
3527 "%s: query attribute, idn %d, failed with error %d after %d retries\n",
3528 __func__, idn, ret, QUERY_REQ_RETRIES);
3529 return ret;
3530 }
3531
__ufshcd_query_descriptor(struct ufs_hba * hba,enum query_opcode opcode,enum desc_idn idn,u8 index,u8 selector,u8 * desc_buf,int * buf_len)3532 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3533 enum query_opcode opcode, enum desc_idn idn, u8 index,
3534 u8 selector, u8 *desc_buf, int *buf_len)
3535 {
3536 struct ufs_query_req *request = NULL;
3537 struct ufs_query_res *response = NULL;
3538 int err;
3539
3540 BUG_ON(!hba);
3541
3542 if (!desc_buf) {
3543 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3544 __func__, opcode);
3545 return -EINVAL;
3546 }
3547
3548 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3549 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3550 __func__, *buf_len);
3551 return -EINVAL;
3552 }
3553
3554 ufshcd_dev_man_lock(hba);
3555
3556 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3557 selector);
3558 hba->dev_cmd.query.descriptor = desc_buf;
3559 request->upiu_req.length = cpu_to_be16(*buf_len);
3560
3561 switch (opcode) {
3562 case UPIU_QUERY_OPCODE_WRITE_DESC:
3563 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3564 break;
3565 case UPIU_QUERY_OPCODE_READ_DESC:
3566 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3567 break;
3568 default:
3569 dev_err(hba->dev,
3570 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3571 __func__, opcode);
3572 err = -EINVAL;
3573 goto out_unlock;
3574 }
3575
3576 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3577
3578 if (err) {
3579 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3580 __func__, opcode, idn, index, err);
3581 goto out_unlock;
3582 }
3583
3584 *buf_len = be16_to_cpu(response->upiu_res.length);
3585
3586 out_unlock:
3587 hba->dev_cmd.query.descriptor = NULL;
3588 ufshcd_dev_man_unlock(hba);
3589 return err;
3590 }
3591
3592 /**
3593 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3594 * @hba: per-adapter instance
3595 * @opcode: attribute opcode
3596 * @idn: attribute idn to access
3597 * @index: index field
3598 * @selector: selector field
3599 * @desc_buf: the buffer that contains the descriptor
3600 * @buf_len: length parameter passed to the device
3601 *
3602 * The buf_len parameter will contain, on return, the length parameter
3603 * received on the response.
3604 *
3605 * Return: 0 for success, non-zero in case of failure.
3606 */
ufshcd_query_descriptor_retry(struct ufs_hba * hba,enum query_opcode opcode,enum desc_idn idn,u8 index,u8 selector,u8 * desc_buf,int * buf_len)3607 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3608 enum query_opcode opcode,
3609 enum desc_idn idn, u8 index,
3610 u8 selector,
3611 u8 *desc_buf, int *buf_len)
3612 {
3613 int err;
3614 int retries;
3615
3616 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3617 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3618 selector, desc_buf, buf_len);
3619 if (!err || err == -EINVAL)
3620 break;
3621 }
3622
3623 return err;
3624 }
3625
3626 /**
3627 * ufshcd_read_desc_param - read the specified descriptor parameter
3628 * @hba: Pointer to adapter instance
3629 * @desc_id: descriptor idn value
3630 * @desc_index: descriptor index
3631 * @param_offset: offset of the parameter to read
3632 * @param_read_buf: pointer to buffer where parameter would be read
3633 * @param_size: sizeof(param_read_buf)
3634 *
3635 * Return: 0 in case of success, non-zero otherwise.
3636 */
ufshcd_read_desc_param(struct ufs_hba * hba,enum desc_idn desc_id,int desc_index,u8 param_offset,u8 * param_read_buf,u8 param_size)3637 int ufshcd_read_desc_param(struct ufs_hba *hba,
3638 enum desc_idn desc_id,
3639 int desc_index,
3640 u8 param_offset,
3641 u8 *param_read_buf,
3642 u8 param_size)
3643 {
3644 int ret;
3645 u8 *desc_buf;
3646 int buff_len = QUERY_DESC_MAX_SIZE;
3647 bool is_kmalloc = true;
3648
3649 /* Safety check */
3650 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3651 return -EINVAL;
3652
3653 /* Check whether we need temp memory */
3654 if (param_offset != 0 || param_size < buff_len) {
3655 desc_buf = kzalloc(buff_len, GFP_KERNEL);
3656 if (!desc_buf)
3657 return -ENOMEM;
3658 } else {
3659 desc_buf = param_read_buf;
3660 is_kmalloc = false;
3661 }
3662
3663 /* Request for full descriptor */
3664 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3665 desc_id, desc_index, 0,
3666 desc_buf, &buff_len);
3667 if (ret) {
3668 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3669 __func__, desc_id, desc_index, param_offset, ret);
3670 goto out;
3671 }
3672
3673 /* Update descriptor length */
3674 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3675
3676 if (param_offset >= buff_len) {
3677 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3678 __func__, param_offset, desc_id, buff_len);
3679 ret = -EINVAL;
3680 goto out;
3681 }
3682
3683 /* Sanity check */
3684 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3685 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3686 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3687 ret = -EINVAL;
3688 goto out;
3689 }
3690
3691 if (is_kmalloc) {
3692 /* Make sure we don't copy more data than available */
3693 if (param_offset >= buff_len)
3694 ret = -EINVAL;
3695 else
3696 memcpy(param_read_buf, &desc_buf[param_offset],
3697 min_t(u32, param_size, buff_len - param_offset));
3698 }
3699 out:
3700 if (is_kmalloc)
3701 kfree(desc_buf);
3702 return ret;
3703 }
3704
3705 /**
3706 * struct uc_string_id - unicode string
3707 *
3708 * @len: size of this descriptor inclusive
3709 * @type: descriptor type
3710 * @uc: unicode string character
3711 */
3712 struct uc_string_id {
3713 u8 len;
3714 u8 type;
3715 wchar_t uc[];
3716 } __packed;
3717
3718 /* replace non-printable or non-ASCII characters with spaces */
ufshcd_remove_non_printable(u8 ch)3719 static inline char ufshcd_remove_non_printable(u8 ch)
3720 {
3721 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3722 }
3723
3724 /**
3725 * ufshcd_read_string_desc - read string descriptor
3726 * @hba: pointer to adapter instance
3727 * @desc_index: descriptor index
3728 * @buf: pointer to buffer where descriptor would be read,
3729 * the caller should free the memory.
3730 * @ascii: if true convert from unicode to ascii characters
3731 * null terminated string.
3732 *
3733 * Return:
3734 * * string size on success.
3735 * * -ENOMEM: on allocation failure
3736 * * -EINVAL: on a wrong parameter
3737 */
ufshcd_read_string_desc(struct ufs_hba * hba,u8 desc_index,u8 ** buf,bool ascii)3738 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3739 u8 **buf, bool ascii)
3740 {
3741 struct uc_string_id *uc_str;
3742 u8 *str;
3743 int ret;
3744
3745 if (!buf)
3746 return -EINVAL;
3747
3748 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3749 if (!uc_str)
3750 return -ENOMEM;
3751
3752 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3753 (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3754 if (ret < 0) {
3755 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3756 QUERY_REQ_RETRIES, ret);
3757 str = NULL;
3758 goto out;
3759 }
3760
3761 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3762 dev_dbg(hba->dev, "String Desc is of zero length\n");
3763 str = NULL;
3764 ret = 0;
3765 goto out;
3766 }
3767
3768 if (ascii) {
3769 ssize_t ascii_len;
3770 int i;
3771 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3772 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3773 str = kzalloc(ascii_len, GFP_KERNEL);
3774 if (!str) {
3775 ret = -ENOMEM;
3776 goto out;
3777 }
3778
3779 /*
3780 * the descriptor contains string in UTF16 format
3781 * we need to convert to utf-8 so it can be displayed
3782 */
3783 ret = utf16s_to_utf8s(uc_str->uc,
3784 uc_str->len - QUERY_DESC_HDR_SIZE,
3785 UTF16_BIG_ENDIAN, str, ascii_len - 1);
3786
3787 /* replace non-printable or non-ASCII characters with spaces */
3788 for (i = 0; i < ret; i++)
3789 str[i] = ufshcd_remove_non_printable(str[i]);
3790
3791 str[ret++] = '\0';
3792
3793 } else {
3794 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3795 if (!str) {
3796 ret = -ENOMEM;
3797 goto out;
3798 }
3799 ret = uc_str->len;
3800 }
3801 out:
3802 *buf = str;
3803 kfree(uc_str);
3804 return ret;
3805 }
3806
3807 /**
3808 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3809 * @hba: Pointer to adapter instance
3810 * @lun: lun id
3811 * @param_offset: offset of the parameter to read
3812 * @param_read_buf: pointer to buffer where parameter would be read
3813 * @param_size: sizeof(param_read_buf)
3814 *
3815 * Return: 0 in case of success, non-zero otherwise.
3816 */
ufshcd_read_unit_desc_param(struct ufs_hba * hba,int lun,enum unit_desc_param param_offset,u8 * param_read_buf,u32 param_size)3817 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3818 int lun,
3819 enum unit_desc_param param_offset,
3820 u8 *param_read_buf,
3821 u32 param_size)
3822 {
3823 /*
3824 * Unit descriptors are only available for general purpose LUs (LUN id
3825 * from 0 to 7) and RPMB Well known LU.
3826 */
3827 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
3828 return -EOPNOTSUPP;
3829
3830 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3831 param_offset, param_read_buf, param_size);
3832 }
3833
ufshcd_get_ref_clk_gating_wait(struct ufs_hba * hba)3834 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3835 {
3836 int err = 0;
3837 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3838
3839 if (hba->dev_info.wspecversion >= 0x300) {
3840 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3841 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3842 &gating_wait);
3843 if (err)
3844 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3845 err, gating_wait);
3846
3847 if (gating_wait == 0) {
3848 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3849 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3850 gating_wait);
3851 }
3852
3853 hba->dev_info.clk_gating_wait_us = gating_wait;
3854 }
3855
3856 return err;
3857 }
3858
3859 /**
3860 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3861 * @hba: per adapter instance
3862 *
3863 * 1. Allocate DMA memory for Command Descriptor array
3864 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3865 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3866 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3867 * (UTMRDL)
3868 * 4. Allocate memory for local reference block(lrb).
3869 *
3870 * Return: 0 for success, non-zero in case of failure.
3871 */
ufshcd_memory_alloc(struct ufs_hba * hba)3872 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3873 {
3874 size_t utmrdl_size, utrdl_size, ucdl_size;
3875
3876 /* Allocate memory for UTP command descriptors */
3877 ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs;
3878 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3879 ucdl_size,
3880 &hba->ucdl_dma_addr,
3881 GFP_KERNEL);
3882
3883 /*
3884 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3885 */
3886 if (!hba->ucdl_base_addr ||
3887 WARN_ON(hba->ucdl_dma_addr & (128 - 1))) {
3888 dev_err(hba->dev,
3889 "Command Descriptor Memory allocation failed\n");
3890 goto out;
3891 }
3892
3893 /*
3894 * Allocate memory for UTP Transfer descriptors
3895 * UFSHCI requires 1KB alignment of UTRD
3896 */
3897 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3898 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3899 utrdl_size,
3900 &hba->utrdl_dma_addr,
3901 GFP_KERNEL);
3902 if (!hba->utrdl_base_addr ||
3903 WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) {
3904 dev_err(hba->dev,
3905 "Transfer Descriptor Memory allocation failed\n");
3906 goto out;
3907 }
3908
3909 /*
3910 * Skip utmrdl allocation; it may have been
3911 * allocated during first pass and not released during
3912 * MCQ memory allocation.
3913 * See ufshcd_release_sdb_queue() and ufshcd_config_mcq()
3914 */
3915 if (hba->utmrdl_base_addr)
3916 goto skip_utmrdl;
3917 /*
3918 * Allocate memory for UTP Task Management descriptors
3919 * UFSHCI requires 1KB alignment of UTMRD
3920 */
3921 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3922 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3923 utmrdl_size,
3924 &hba->utmrdl_dma_addr,
3925 GFP_KERNEL);
3926 if (!hba->utmrdl_base_addr ||
3927 WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) {
3928 dev_err(hba->dev,
3929 "Task Management Descriptor Memory allocation failed\n");
3930 goto out;
3931 }
3932
3933 skip_utmrdl:
3934 /* Allocate memory for local reference block */
3935 hba->lrb = devm_kcalloc(hba->dev,
3936 hba->nutrs, sizeof(struct ufshcd_lrb),
3937 GFP_KERNEL);
3938 if (!hba->lrb) {
3939 dev_err(hba->dev, "LRB Memory allocation failed\n");
3940 goto out;
3941 }
3942 return 0;
3943 out:
3944 return -ENOMEM;
3945 }
3946
3947 /**
3948 * ufshcd_host_memory_configure - configure local reference block with
3949 * memory offsets
3950 * @hba: per adapter instance
3951 *
3952 * Configure Host memory space
3953 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3954 * address.
3955 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3956 * and PRDT offset.
3957 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3958 * into local reference block.
3959 */
ufshcd_host_memory_configure(struct ufs_hba * hba)3960 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3961 {
3962 struct utp_transfer_req_desc *utrdlp;
3963 dma_addr_t cmd_desc_dma_addr;
3964 dma_addr_t cmd_desc_element_addr;
3965 u16 response_offset;
3966 u16 prdt_offset;
3967 int cmd_desc_size;
3968 int i;
3969
3970 utrdlp = hba->utrdl_base_addr;
3971
3972 response_offset =
3973 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3974 prdt_offset =
3975 offsetof(struct utp_transfer_cmd_desc, prd_table);
3976
3977 cmd_desc_size = ufshcd_get_ucd_size(hba);
3978 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3979
3980 for (i = 0; i < hba->nutrs; i++) {
3981 /* Configure UTRD with command descriptor base address */
3982 cmd_desc_element_addr =
3983 (cmd_desc_dma_addr + (cmd_desc_size * i));
3984 utrdlp[i].command_desc_base_addr =
3985 cpu_to_le64(cmd_desc_element_addr);
3986
3987 /* Response upiu and prdt offset should be in double words */
3988 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3989 utrdlp[i].response_upiu_offset =
3990 cpu_to_le16(response_offset);
3991 utrdlp[i].prd_table_offset =
3992 cpu_to_le16(prdt_offset);
3993 utrdlp[i].response_upiu_length =
3994 cpu_to_le16(ALIGNED_UPIU_SIZE);
3995 } else {
3996 utrdlp[i].response_upiu_offset =
3997 cpu_to_le16(response_offset >> 2);
3998 utrdlp[i].prd_table_offset =
3999 cpu_to_le16(prdt_offset >> 2);
4000 utrdlp[i].response_upiu_length =
4001 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
4002 }
4003
4004 ufshcd_init_lrb(hba, &hba->lrb[i], i);
4005 }
4006 }
4007
4008 /**
4009 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
4010 * @hba: per adapter instance
4011 *
4012 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
4013 * in order to initialize the Unipro link startup procedure.
4014 * Once the Unipro links are up, the device connected to the controller
4015 * is detected.
4016 *
4017 * Return: 0 on success, non-zero value on failure.
4018 */
ufshcd_dme_link_startup(struct ufs_hba * hba)4019 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
4020 {
4021 struct uic_command uic_cmd = {
4022 .command = UIC_CMD_DME_LINK_STARTUP,
4023 };
4024 int ret;
4025
4026 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4027 if (ret)
4028 dev_dbg(hba->dev,
4029 "dme-link-startup: error code %d\n", ret);
4030 return ret;
4031 }
4032 /**
4033 * ufshcd_dme_reset - UIC command for DME_RESET
4034 * @hba: per adapter instance
4035 *
4036 * DME_RESET command is issued in order to reset UniPro stack.
4037 * This function now deals with cold reset.
4038 *
4039 * Return: 0 on success, non-zero value on failure.
4040 */
ufshcd_dme_reset(struct ufs_hba * hba)4041 static int ufshcd_dme_reset(struct ufs_hba *hba)
4042 {
4043 struct uic_command uic_cmd = {
4044 .command = UIC_CMD_DME_RESET,
4045 };
4046 int ret;
4047
4048 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4049 if (ret)
4050 dev_err(hba->dev,
4051 "dme-reset: error code %d\n", ret);
4052
4053 return ret;
4054 }
4055
ufshcd_dme_configure_adapt(struct ufs_hba * hba,int agreed_gear,int adapt_val)4056 int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
4057 int agreed_gear,
4058 int adapt_val)
4059 {
4060 int ret;
4061
4062 if (agreed_gear < UFS_HS_G4)
4063 adapt_val = PA_NO_ADAPT;
4064
4065 ret = ufshcd_dme_set(hba,
4066 UIC_ARG_MIB(PA_TXHSADAPTTYPE),
4067 adapt_val);
4068 return ret;
4069 }
4070 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
4071
4072 /**
4073 * ufshcd_dme_enable - UIC command for DME_ENABLE
4074 * @hba: per adapter instance
4075 *
4076 * DME_ENABLE command is issued in order to enable UniPro stack.
4077 *
4078 * Return: 0 on success, non-zero value on failure.
4079 */
ufshcd_dme_enable(struct ufs_hba * hba)4080 static int ufshcd_dme_enable(struct ufs_hba *hba)
4081 {
4082 struct uic_command uic_cmd = {
4083 .command = UIC_CMD_DME_ENABLE,
4084 };
4085 int ret;
4086
4087 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4088 if (ret)
4089 dev_err(hba->dev,
4090 "dme-enable: error code %d\n", ret);
4091
4092 return ret;
4093 }
4094
ufshcd_add_delay_before_dme_cmd(struct ufs_hba * hba)4095 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
4096 {
4097 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
4098 unsigned long min_sleep_time_us;
4099
4100 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
4101 return;
4102
4103 /*
4104 * last_dme_cmd_tstamp will be 0 only for 1st call to
4105 * this function
4106 */
4107 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
4108 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
4109 } else {
4110 unsigned long delta =
4111 (unsigned long) ktime_to_us(
4112 ktime_sub(ktime_get(),
4113 hba->last_dme_cmd_tstamp));
4114
4115 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
4116 min_sleep_time_us =
4117 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
4118 else
4119 min_sleep_time_us = 0; /* no more delay required */
4120 }
4121
4122 if (min_sleep_time_us > 0) {
4123 /* allow sleep for extra 50us if needed */
4124 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
4125 }
4126
4127 /* update the last_dme_cmd_tstamp */
4128 hba->last_dme_cmd_tstamp = ktime_get();
4129 }
4130
4131 /**
4132 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
4133 * @hba: per adapter instance
4134 * @attr_sel: uic command argument1
4135 * @attr_set: attribute set type as uic command argument2
4136 * @mib_val: setting value as uic command argument3
4137 * @peer: indicate whether peer or local
4138 *
4139 * Return: 0 on success, non-zero value on failure.
4140 */
ufshcd_dme_set_attr(struct ufs_hba * hba,u32 attr_sel,u8 attr_set,u32 mib_val,u8 peer)4141 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
4142 u8 attr_set, u32 mib_val, u8 peer)
4143 {
4144 struct uic_command uic_cmd = {
4145 .command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET,
4146 .argument1 = attr_sel,
4147 .argument2 = UIC_ARG_ATTR_TYPE(attr_set),
4148 .argument3 = mib_val,
4149 };
4150 static const char *const action[] = {
4151 "dme-set",
4152 "dme-peer-set"
4153 };
4154 const char *set = action[!!peer];
4155 int ret;
4156 int retries = UFS_UIC_COMMAND_RETRIES;
4157
4158 do {
4159 /* for peer attributes we retry upon failure */
4160 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4161 if (ret)
4162 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
4163 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
4164 } while (ret && peer && --retries);
4165
4166 if (ret)
4167 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
4168 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
4169 UFS_UIC_COMMAND_RETRIES - retries);
4170
4171 return ret;
4172 }
4173 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
4174
4175 /**
4176 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
4177 * @hba: per adapter instance
4178 * @attr_sel: uic command argument1
4179 * @mib_val: the value of the attribute as returned by the UIC command
4180 * @peer: indicate whether peer or local
4181 *
4182 * Return: 0 on success, non-zero value on failure.
4183 */
ufshcd_dme_get_attr(struct ufs_hba * hba,u32 attr_sel,u32 * mib_val,u8 peer)4184 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
4185 u32 *mib_val, u8 peer)
4186 {
4187 struct uic_command uic_cmd = {
4188 .command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET,
4189 .argument1 = attr_sel,
4190 };
4191 static const char *const action[] = {
4192 "dme-get",
4193 "dme-peer-get"
4194 };
4195 const char *get = action[!!peer];
4196 int ret;
4197 int retries = UFS_UIC_COMMAND_RETRIES;
4198 struct ufs_pa_layer_attr orig_pwr_info;
4199 struct ufs_pa_layer_attr temp_pwr_info;
4200 bool pwr_mode_change = false;
4201
4202 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
4203 orig_pwr_info = hba->pwr_info;
4204 temp_pwr_info = orig_pwr_info;
4205
4206 if (orig_pwr_info.pwr_tx == FAST_MODE ||
4207 orig_pwr_info.pwr_rx == FAST_MODE) {
4208 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
4209 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
4210 pwr_mode_change = true;
4211 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
4212 orig_pwr_info.pwr_rx == SLOW_MODE) {
4213 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
4214 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
4215 pwr_mode_change = true;
4216 }
4217 if (pwr_mode_change) {
4218 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
4219 if (ret)
4220 goto out;
4221 }
4222 }
4223
4224 do {
4225 /* for peer attributes we retry upon failure */
4226 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4227 if (ret)
4228 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
4229 get, UIC_GET_ATTR_ID(attr_sel), ret);
4230 } while (ret && peer && --retries);
4231
4232 if (ret)
4233 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
4234 get, UIC_GET_ATTR_ID(attr_sel),
4235 UFS_UIC_COMMAND_RETRIES - retries);
4236
4237 if (mib_val && !ret)
4238 *mib_val = uic_cmd.argument3;
4239
4240 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
4241 && pwr_mode_change)
4242 ufshcd_change_power_mode(hba, &orig_pwr_info);
4243 out:
4244 return ret;
4245 }
4246 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
4247
4248 /**
4249 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
4250 * state) and waits for it to take effect.
4251 *
4252 * @hba: per adapter instance
4253 * @cmd: UIC command to execute
4254 *
4255 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
4256 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
4257 * and device UniPro link and hence it's final completion would be indicated by
4258 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
4259 * addition to normal UIC command completion Status (UCCS). This function only
4260 * returns after the relevant status bits indicate the completion.
4261 *
4262 * Return: 0 on success, non-zero value on failure.
4263 */
ufshcd_uic_pwr_ctrl(struct ufs_hba * hba,struct uic_command * cmd)4264 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
4265 {
4266 DECLARE_COMPLETION_ONSTACK(uic_async_done);
4267 unsigned long flags;
4268 u8 status;
4269 int ret;
4270 bool reenable_intr = false;
4271
4272 mutex_lock(&hba->uic_cmd_mutex);
4273 ufshcd_add_delay_before_dme_cmd(hba);
4274
4275 spin_lock_irqsave(hba->host->host_lock, flags);
4276 if (ufshcd_is_link_broken(hba)) {
4277 ret = -ENOLINK;
4278 goto out_unlock;
4279 }
4280 hba->uic_async_done = &uic_async_done;
4281 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
4282 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
4283 /*
4284 * Make sure UIC command completion interrupt is disabled before
4285 * issuing UIC command.
4286 */
4287 ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
4288 reenable_intr = true;
4289 }
4290 spin_unlock_irqrestore(hba->host->host_lock, flags);
4291 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
4292 if (ret) {
4293 dev_err(hba->dev,
4294 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
4295 cmd->command, cmd->argument3, ret);
4296 goto out;
4297 }
4298
4299 if (!wait_for_completion_timeout(hba->uic_async_done,
4300 msecs_to_jiffies(uic_cmd_timeout))) {
4301 dev_err(hba->dev,
4302 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
4303 cmd->command, cmd->argument3);
4304
4305 if (!cmd->cmd_active) {
4306 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
4307 __func__);
4308 goto check_upmcrs;
4309 }
4310
4311 ret = -ETIMEDOUT;
4312 goto out;
4313 }
4314
4315 check_upmcrs:
4316 status = ufshcd_get_upmcrs(hba);
4317 if (status != PWR_LOCAL) {
4318 dev_err(hba->dev,
4319 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
4320 cmd->command, status);
4321 ret = (status != PWR_OK) ? status : -1;
4322 }
4323 out:
4324 if (ret) {
4325 ufshcd_print_host_state(hba);
4326 ufshcd_print_pwr_info(hba);
4327 ufshcd_print_evt_hist(hba);
4328 }
4329
4330 spin_lock_irqsave(hba->host->host_lock, flags);
4331 hba->active_uic_cmd = NULL;
4332 hba->uic_async_done = NULL;
4333 if (reenable_intr)
4334 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4335 if (ret) {
4336 ufshcd_set_link_broken(hba);
4337 ufshcd_schedule_eh_work(hba);
4338 }
4339 out_unlock:
4340 spin_unlock_irqrestore(hba->host->host_lock, flags);
4341 mutex_unlock(&hba->uic_cmd_mutex);
4342
4343 return ret;
4344 }
4345
4346 /**
4347 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4348 * using DME_SET primitives.
4349 * @hba: per adapter instance
4350 * @mode: powr mode value
4351 *
4352 * Return: 0 on success, non-zero value on failure.
4353 */
ufshcd_uic_change_pwr_mode(struct ufs_hba * hba,u8 mode)4354 int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4355 {
4356 struct uic_command uic_cmd = {
4357 .command = UIC_CMD_DME_SET,
4358 .argument1 = UIC_ARG_MIB(PA_PWRMODE),
4359 .argument3 = mode,
4360 };
4361 int ret;
4362
4363 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4364 ret = ufshcd_dme_set(hba,
4365 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4366 if (ret) {
4367 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4368 __func__, ret);
4369 goto out;
4370 }
4371 }
4372
4373 ufshcd_hold(hba);
4374 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4375 ufshcd_release(hba);
4376
4377 out:
4378 return ret;
4379 }
4380 EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode);
4381
ufshcd_link_recovery(struct ufs_hba * hba)4382 int ufshcd_link_recovery(struct ufs_hba *hba)
4383 {
4384 int ret;
4385 unsigned long flags;
4386
4387 spin_lock_irqsave(hba->host->host_lock, flags);
4388 hba->ufshcd_state = UFSHCD_STATE_RESET;
4389 ufshcd_set_eh_in_progress(hba);
4390 spin_unlock_irqrestore(hba->host->host_lock, flags);
4391
4392 /* Reset the attached device */
4393 ufshcd_device_reset(hba);
4394
4395 ret = ufshcd_host_reset_and_restore(hba);
4396
4397 spin_lock_irqsave(hba->host->host_lock, flags);
4398 if (ret)
4399 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4400 ufshcd_clear_eh_in_progress(hba);
4401 spin_unlock_irqrestore(hba->host->host_lock, flags);
4402
4403 if (ret)
4404 dev_err(hba->dev, "%s: link recovery failed, err %d",
4405 __func__, ret);
4406
4407 return ret;
4408 }
4409 EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
4410
ufshcd_uic_hibern8_enter(struct ufs_hba * hba)4411 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
4412 {
4413 struct uic_command uic_cmd = {
4414 .command = UIC_CMD_DME_HIBER_ENTER,
4415 };
4416 ktime_t start = ktime_get();
4417 int ret;
4418
4419 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4420
4421 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4422 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4423 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4424
4425 if (ret)
4426 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4427 __func__, ret);
4428 else
4429 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4430 POST_CHANGE);
4431
4432 return ret;
4433 }
4434 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter);
4435
ufshcd_uic_hibern8_exit(struct ufs_hba * hba)4436 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
4437 {
4438 struct uic_command uic_cmd = {
4439 .command = UIC_CMD_DME_HIBER_EXIT,
4440 };
4441 int ret;
4442 ktime_t start = ktime_get();
4443
4444 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4445
4446 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4447 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4448 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4449
4450 if (ret) {
4451 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4452 __func__, ret);
4453 } else {
4454 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4455 POST_CHANGE);
4456 hba->ufs_stats.last_hibern8_exit_tstamp = local_clock();
4457 hba->ufs_stats.hibern8_exit_cnt++;
4458 }
4459
4460 return ret;
4461 }
4462 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
4463
ufshcd_configure_auto_hibern8(struct ufs_hba * hba)4464 static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba)
4465 {
4466 if (!ufshcd_is_auto_hibern8_supported(hba))
4467 return;
4468
4469 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4470 }
4471
ufshcd_auto_hibern8_update(struct ufs_hba * hba,u32 ahit)4472 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4473 {
4474 const u32 cur_ahit = READ_ONCE(hba->ahit);
4475
4476 if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit)
4477 return;
4478
4479 WRITE_ONCE(hba->ahit, ahit);
4480 if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) {
4481 ufshcd_rpm_get_sync(hba);
4482 ufshcd_hold(hba);
4483 ufshcd_configure_auto_hibern8(hba);
4484 ufshcd_release(hba);
4485 ufshcd_rpm_put_sync(hba);
4486 }
4487 }
4488 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4489
4490 /**
4491 * ufshcd_init_pwr_info - setting the POR (power on reset)
4492 * values in hba power info
4493 * @hba: per-adapter instance
4494 */
ufshcd_init_pwr_info(struct ufs_hba * hba)4495 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4496 {
4497 hba->pwr_info.gear_rx = UFS_PWM_G1;
4498 hba->pwr_info.gear_tx = UFS_PWM_G1;
4499 hba->pwr_info.lane_rx = UFS_LANE_1;
4500 hba->pwr_info.lane_tx = UFS_LANE_1;
4501 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4502 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4503 hba->pwr_info.hs_rate = 0;
4504 }
4505
4506 /**
4507 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4508 * @hba: per-adapter instance
4509 *
4510 * Return: 0 upon success; < 0 upon failure.
4511 */
ufshcd_get_max_pwr_mode(struct ufs_hba * hba)4512 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4513 {
4514 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4515
4516 if (hba->max_pwr_info.is_valid)
4517 return 0;
4518
4519 if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) {
4520 pwr_info->pwr_tx = FASTAUTO_MODE;
4521 pwr_info->pwr_rx = FASTAUTO_MODE;
4522 } else {
4523 pwr_info->pwr_tx = FAST_MODE;
4524 pwr_info->pwr_rx = FAST_MODE;
4525 }
4526 pwr_info->hs_rate = PA_HS_MODE_B;
4527
4528 /* Get the connected lane count */
4529 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4530 &pwr_info->lane_rx);
4531 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4532 &pwr_info->lane_tx);
4533
4534 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4535 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4536 __func__,
4537 pwr_info->lane_rx,
4538 pwr_info->lane_tx);
4539 return -EINVAL;
4540 }
4541
4542 /*
4543 * First, get the maximum gears of HS speed.
4544 * If a zero value, it means there is no HSGEAR capability.
4545 * Then, get the maximum gears of PWM speed.
4546 */
4547 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4548 if (!pwr_info->gear_rx) {
4549 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4550 &pwr_info->gear_rx);
4551 if (!pwr_info->gear_rx) {
4552 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4553 __func__, pwr_info->gear_rx);
4554 return -EINVAL;
4555 }
4556 pwr_info->pwr_rx = SLOW_MODE;
4557 }
4558
4559 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4560 &pwr_info->gear_tx);
4561 if (!pwr_info->gear_tx) {
4562 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4563 &pwr_info->gear_tx);
4564 if (!pwr_info->gear_tx) {
4565 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4566 __func__, pwr_info->gear_tx);
4567 return -EINVAL;
4568 }
4569 pwr_info->pwr_tx = SLOW_MODE;
4570 }
4571
4572 hba->max_pwr_info.is_valid = true;
4573 return 0;
4574 }
4575
ufshcd_change_power_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode)4576 static int ufshcd_change_power_mode(struct ufs_hba *hba,
4577 struct ufs_pa_layer_attr *pwr_mode)
4578 {
4579 int ret;
4580
4581 /* if already configured to the requested pwr_mode */
4582 if (!hba->force_pmc &&
4583 pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4584 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4585 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4586 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4587 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4588 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4589 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4590 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4591 return 0;
4592 }
4593
4594 /*
4595 * Configure attributes for power mode change with below.
4596 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4597 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4598 * - PA_HSSERIES
4599 */
4600 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4601 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4602 pwr_mode->lane_rx);
4603 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4604 pwr_mode->pwr_rx == FAST_MODE)
4605 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true);
4606 else
4607 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false);
4608
4609 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4610 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4611 pwr_mode->lane_tx);
4612 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4613 pwr_mode->pwr_tx == FAST_MODE)
4614 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
4615 else
4616 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false);
4617
4618 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4619 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4620 pwr_mode->pwr_rx == FAST_MODE ||
4621 pwr_mode->pwr_tx == FAST_MODE)
4622 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4623 pwr_mode->hs_rate);
4624
4625 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4626 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4627 DL_FC0ProtectionTimeOutVal_Default);
4628 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4629 DL_TC0ReplayTimeOutVal_Default);
4630 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4631 DL_AFC0ReqTimeOutVal_Default);
4632 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4633 DL_FC1ProtectionTimeOutVal_Default);
4634 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4635 DL_TC1ReplayTimeOutVal_Default);
4636 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4637 DL_AFC1ReqTimeOutVal_Default);
4638
4639 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4640 DL_FC0ProtectionTimeOutVal_Default);
4641 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4642 DL_TC0ReplayTimeOutVal_Default);
4643 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4644 DL_AFC0ReqTimeOutVal_Default);
4645 }
4646
4647 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4648 | pwr_mode->pwr_tx);
4649
4650 if (ret) {
4651 dev_err(hba->dev,
4652 "%s: power mode change failed %d\n", __func__, ret);
4653 } else {
4654 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4655 pwr_mode);
4656
4657 memcpy(&hba->pwr_info, pwr_mode,
4658 sizeof(struct ufs_pa_layer_attr));
4659 }
4660
4661 return ret;
4662 }
4663
4664 /**
4665 * ufshcd_config_pwr_mode - configure a new power mode
4666 * @hba: per-adapter instance
4667 * @desired_pwr_mode: desired power configuration
4668 *
4669 * Return: 0 upon success; < 0 upon failure.
4670 */
ufshcd_config_pwr_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * desired_pwr_mode)4671 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4672 struct ufs_pa_layer_attr *desired_pwr_mode)
4673 {
4674 struct ufs_pa_layer_attr final_params = { 0 };
4675 int ret;
4676
4677 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4678 desired_pwr_mode, &final_params);
4679
4680 if (ret)
4681 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4682
4683 ret = ufshcd_change_power_mode(hba, &final_params);
4684
4685 return ret;
4686 }
4687 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4688
4689 /**
4690 * ufshcd_complete_dev_init() - checks device readiness
4691 * @hba: per-adapter instance
4692 *
4693 * Set fDeviceInit flag and poll until device toggles it.
4694 *
4695 * Return: 0 upon success; < 0 upon failure.
4696 */
ufshcd_complete_dev_init(struct ufs_hba * hba)4697 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4698 {
4699 int err;
4700 bool flag_res = true;
4701 ktime_t timeout;
4702
4703 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4704 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4705 if (err) {
4706 dev_err(hba->dev,
4707 "%s: setting fDeviceInit flag failed with error %d\n",
4708 __func__, err);
4709 goto out;
4710 }
4711
4712 /* Poll fDeviceInit flag to be cleared */
4713 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4714 do {
4715 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4716 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4717 if (!flag_res)
4718 break;
4719 usleep_range(500, 1000);
4720 } while (ktime_before(ktime_get(), timeout));
4721
4722 if (err) {
4723 dev_err(hba->dev,
4724 "%s: reading fDeviceInit flag failed with error %d\n",
4725 __func__, err);
4726 } else if (flag_res) {
4727 dev_err(hba->dev,
4728 "%s: fDeviceInit was not cleared by the device\n",
4729 __func__);
4730 err = -EBUSY;
4731 }
4732 out:
4733 return err;
4734 }
4735
4736 /**
4737 * ufshcd_make_hba_operational - Make UFS controller operational
4738 * @hba: per adapter instance
4739 *
4740 * To bring UFS host controller to operational state,
4741 * 1. Enable required interrupts
4742 * 2. Configure interrupt aggregation
4743 * 3. Program UTRL and UTMRL base address
4744 * 4. Configure run-stop-registers
4745 *
4746 * Return: 0 on success, non-zero value on failure.
4747 */
ufshcd_make_hba_operational(struct ufs_hba * hba)4748 int ufshcd_make_hba_operational(struct ufs_hba *hba)
4749 {
4750 int err = 0;
4751 u32 reg;
4752
4753 /* Enable required interrupts */
4754 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4755
4756 /* Configure interrupt aggregation */
4757 if (ufshcd_is_intr_aggr_allowed(hba))
4758 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4759 else
4760 ufshcd_disable_intr_aggr(hba);
4761
4762 /* Configure UTRL and UTMRL base address registers */
4763 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4764 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4765 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4766 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4767 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4768 REG_UTP_TASK_REQ_LIST_BASE_L);
4769 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4770 REG_UTP_TASK_REQ_LIST_BASE_H);
4771
4772 /*
4773 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4774 */
4775 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4776 if (!(ufshcd_get_lists_status(reg))) {
4777 ufshcd_enable_run_stop_reg(hba);
4778 } else {
4779 dev_err(hba->dev,
4780 "Host controller not ready to process requests");
4781 err = -EIO;
4782 }
4783
4784 return err;
4785 }
4786 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4787
4788 /**
4789 * ufshcd_hba_stop - Send controller to reset state
4790 * @hba: per adapter instance
4791 */
ufshcd_hba_stop(struct ufs_hba * hba)4792 void ufshcd_hba_stop(struct ufs_hba *hba)
4793 {
4794 unsigned long flags;
4795 int err;
4796
4797 /*
4798 * Obtain the host lock to prevent that the controller is disabled
4799 * while the UFS interrupt handler is active on another CPU.
4800 */
4801 spin_lock_irqsave(hba->host->host_lock, flags);
4802 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
4803 spin_unlock_irqrestore(hba->host->host_lock, flags);
4804
4805 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4806 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4807 10, 1);
4808 if (err)
4809 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4810 }
4811 EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
4812
4813 /**
4814 * ufshcd_hba_execute_hce - initialize the controller
4815 * @hba: per adapter instance
4816 *
4817 * The controller resets itself and controller firmware initialization
4818 * sequence kicks off. When controller is ready it will set
4819 * the Host Controller Enable bit to 1.
4820 *
4821 * Return: 0 on success, non-zero value on failure.
4822 */
ufshcd_hba_execute_hce(struct ufs_hba * hba)4823 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4824 {
4825 int retry_outer = 3;
4826 int retry_inner;
4827
4828 start:
4829 if (ufshcd_is_hba_active(hba))
4830 /* change controller state to "reset state" */
4831 ufshcd_hba_stop(hba);
4832
4833 /* UniPro link is disabled at this point */
4834 ufshcd_set_link_off(hba);
4835
4836 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4837
4838 /* start controller initialization sequence */
4839 ufshcd_hba_start(hba);
4840
4841 /*
4842 * To initialize a UFS host controller HCE bit must be set to 1.
4843 * During initialization the HCE bit value changes from 1->0->1.
4844 * When the host controller completes initialization sequence
4845 * it sets the value of HCE bit to 1. The same HCE bit is read back
4846 * to check if the controller has completed initialization sequence.
4847 * So without this delay the value HCE = 1, set in the previous
4848 * instruction might be read back.
4849 * This delay can be changed based on the controller.
4850 */
4851 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4852
4853 /* wait for the host controller to complete initialization */
4854 retry_inner = 50;
4855 while (!ufshcd_is_hba_active(hba)) {
4856 if (retry_inner) {
4857 retry_inner--;
4858 } else {
4859 dev_err(hba->dev,
4860 "Controller enable failed\n");
4861 if (retry_outer) {
4862 retry_outer--;
4863 goto start;
4864 }
4865 return -EIO;
4866 }
4867 usleep_range(1000, 1100);
4868 }
4869
4870 /* enable UIC related interrupts */
4871 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4872
4873 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4874
4875 return 0;
4876 }
4877
ufshcd_hba_enable(struct ufs_hba * hba)4878 int ufshcd_hba_enable(struct ufs_hba *hba)
4879 {
4880 int ret;
4881
4882 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4883 ufshcd_set_link_off(hba);
4884 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4885
4886 /* enable UIC related interrupts */
4887 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4888 ret = ufshcd_dme_reset(hba);
4889 if (ret) {
4890 dev_err(hba->dev, "DME_RESET failed\n");
4891 return ret;
4892 }
4893
4894 ret = ufshcd_dme_enable(hba);
4895 if (ret) {
4896 dev_err(hba->dev, "Enabling DME failed\n");
4897 return ret;
4898 }
4899
4900 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4901 } else {
4902 ret = ufshcd_hba_execute_hce(hba);
4903 }
4904
4905 return ret;
4906 }
4907 EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4908
ufshcd_disable_tx_lcc(struct ufs_hba * hba,bool peer)4909 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4910 {
4911 int tx_lanes = 0, i, err = 0;
4912
4913 if (!peer)
4914 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4915 &tx_lanes);
4916 else
4917 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4918 &tx_lanes);
4919 for (i = 0; i < tx_lanes; i++) {
4920 if (!peer)
4921 err = ufshcd_dme_set(hba,
4922 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4923 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4924 0);
4925 else
4926 err = ufshcd_dme_peer_set(hba,
4927 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4928 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4929 0);
4930 if (err) {
4931 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4932 __func__, peer, i, err);
4933 break;
4934 }
4935 }
4936
4937 return err;
4938 }
4939
ufshcd_disable_device_tx_lcc(struct ufs_hba * hba)4940 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4941 {
4942 return ufshcd_disable_tx_lcc(hba, true);
4943 }
4944
ufshcd_update_evt_hist(struct ufs_hba * hba,u32 id,u32 val)4945 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
4946 {
4947 struct ufs_event_hist *e;
4948
4949 if (id >= UFS_EVT_CNT)
4950 return;
4951
4952 e = &hba->ufs_stats.event[id];
4953 e->val[e->pos] = val;
4954 e->tstamp[e->pos] = local_clock();
4955 e->cnt += 1;
4956 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4957
4958 ufshcd_vops_event_notify(hba, id, &val);
4959 }
4960 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
4961
4962 /**
4963 * ufshcd_link_startup - Initialize unipro link startup
4964 * @hba: per adapter instance
4965 *
4966 * Return: 0 for success, non-zero in case of failure.
4967 */
ufshcd_link_startup(struct ufs_hba * hba)4968 static int ufshcd_link_startup(struct ufs_hba *hba)
4969 {
4970 int ret;
4971 int retries = DME_LINKSTARTUP_RETRIES;
4972 bool link_startup_again = false;
4973
4974 /*
4975 * If UFS device isn't active then we will have to issue link startup
4976 * 2 times to make sure the device state move to active.
4977 */
4978 if (!ufshcd_is_ufs_dev_active(hba))
4979 link_startup_again = true;
4980
4981 link_startup:
4982 do {
4983 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4984
4985 ret = ufshcd_dme_link_startup(hba);
4986
4987 /* check if device is detected by inter-connect layer */
4988 if (!ret && !ufshcd_is_device_present(hba)) {
4989 ufshcd_update_evt_hist(hba,
4990 UFS_EVT_LINK_STARTUP_FAIL,
4991 0);
4992 dev_err(hba->dev, "%s: Device not present\n", __func__);
4993 ret = -ENXIO;
4994 goto out;
4995 }
4996
4997 /*
4998 * DME link lost indication is only received when link is up,
4999 * but we can't be sure if the link is up until link startup
5000 * succeeds. So reset the local Uni-Pro and try again.
5001 */
5002 if (ret && retries && ufshcd_hba_enable(hba)) {
5003 ufshcd_update_evt_hist(hba,
5004 UFS_EVT_LINK_STARTUP_FAIL,
5005 (u32)ret);
5006 goto out;
5007 }
5008 } while (ret && retries--);
5009
5010 if (ret) {
5011 /* failed to get the link up... retire */
5012 ufshcd_update_evt_hist(hba,
5013 UFS_EVT_LINK_STARTUP_FAIL,
5014 (u32)ret);
5015 goto out;
5016 }
5017
5018 if (link_startup_again) {
5019 link_startup_again = false;
5020 retries = DME_LINKSTARTUP_RETRIES;
5021 goto link_startup;
5022 }
5023
5024 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
5025 ufshcd_init_pwr_info(hba);
5026 ufshcd_print_pwr_info(hba);
5027
5028 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
5029 ret = ufshcd_disable_device_tx_lcc(hba);
5030 if (ret)
5031 goto out;
5032 }
5033
5034 /* Include any host controller configuration via UIC commands */
5035 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
5036 if (ret)
5037 goto out;
5038
5039 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
5040 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5041 ret = ufshcd_make_hba_operational(hba);
5042 out:
5043 if (ret) {
5044 dev_err(hba->dev, "link startup failed %d\n", ret);
5045 ufshcd_print_host_state(hba);
5046 ufshcd_print_pwr_info(hba);
5047 ufshcd_print_evt_hist(hba);
5048 }
5049 return ret;
5050 }
5051
5052 /**
5053 * ufshcd_verify_dev_init() - Verify device initialization
5054 * @hba: per-adapter instance
5055 *
5056 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
5057 * device Transport Protocol (UTP) layer is ready after a reset.
5058 * If the UTP layer at the device side is not initialized, it may
5059 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
5060 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
5061 *
5062 * Return: 0 upon success; < 0 upon failure.
5063 */
ufshcd_verify_dev_init(struct ufs_hba * hba)5064 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
5065 {
5066 int err = 0;
5067 int retries;
5068
5069 ufshcd_dev_man_lock(hba);
5070
5071 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
5072 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
5073 hba->nop_out_timeout);
5074
5075 if (!err || err == -ETIMEDOUT)
5076 break;
5077
5078 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
5079 }
5080
5081 ufshcd_dev_man_unlock(hba);
5082
5083 if (err)
5084 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
5085 return err;
5086 }
5087
5088 /**
5089 * ufshcd_setup_links - associate link b/w device wlun and other luns
5090 * @sdev: pointer to SCSI device
5091 * @hba: pointer to ufs hba
5092 */
ufshcd_setup_links(struct ufs_hba * hba,struct scsi_device * sdev)5093 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev)
5094 {
5095 struct device_link *link;
5096
5097 /*
5098 * Device wlun is the supplier & rest of the luns are consumers.
5099 * This ensures that device wlun suspends after all other luns.
5100 */
5101 if (hba->ufs_device_wlun) {
5102 link = device_link_add(&sdev->sdev_gendev,
5103 &hba->ufs_device_wlun->sdev_gendev,
5104 DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE);
5105 if (!link) {
5106 dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
5107 dev_name(&hba->ufs_device_wlun->sdev_gendev));
5108 return;
5109 }
5110 hba->luns_avail--;
5111 /* Ignore REPORT_LUN wlun probing */
5112 if (hba->luns_avail == 1) {
5113 ufshcd_rpm_put(hba);
5114 return;
5115 }
5116 } else {
5117 /*
5118 * Device wlun is probed. The assumption is that WLUNs are
5119 * scanned before other LUNs.
5120 */
5121 hba->luns_avail--;
5122 }
5123 }
5124
5125 /**
5126 * ufshcd_lu_init - Initialize the relevant parameters of the LU
5127 * @hba: per-adapter instance
5128 * @sdev: pointer to SCSI device
5129 */
ufshcd_lu_init(struct ufs_hba * hba,struct scsi_device * sdev)5130 static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev)
5131 {
5132 int len = QUERY_DESC_MAX_SIZE;
5133 u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun);
5134 u8 lun_qdepth = hba->nutrs;
5135 u8 *desc_buf;
5136 int ret;
5137
5138 desc_buf = kzalloc(len, GFP_KERNEL);
5139 if (!desc_buf)
5140 goto set_qdepth;
5141
5142 ret = ufshcd_read_unit_desc_param(hba, lun, 0, desc_buf, len);
5143 if (ret < 0) {
5144 if (ret == -EOPNOTSUPP)
5145 /* If LU doesn't support unit descriptor, its queue depth is set to 1 */
5146 lun_qdepth = 1;
5147 kfree(desc_buf);
5148 goto set_qdepth;
5149 }
5150
5151 if (desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]) {
5152 /*
5153 * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will
5154 * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth
5155 */
5156 lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs);
5157 }
5158 /*
5159 * According to UFS device specification, the write protection mode is only supported by
5160 * normal LU, not supported by WLUN.
5161 */
5162 if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported &&
5163 !hba->dev_info.is_lu_power_on_wp &&
5164 desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP)
5165 hba->dev_info.is_lu_power_on_wp = true;
5166
5167 /* In case of RPMB LU, check if advanced RPMB mode is enabled */
5168 if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN &&
5169 desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4))
5170 hba->dev_info.b_advanced_rpmb_en = true;
5171
5172
5173 kfree(desc_buf);
5174 set_qdepth:
5175 /*
5176 * For WLUNs that don't support unit descriptor, queue depth is set to 1. For LUs whose
5177 * bLUQueueDepth == 0, the queue depth is set to a maximum value that host can queue.
5178 */
5179 dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth);
5180 scsi_change_queue_depth(sdev, lun_qdepth);
5181 }
5182
5183 /**
5184 * ufshcd_slave_alloc - handle initial SCSI device configurations
5185 * @sdev: pointer to SCSI device
5186 *
5187 * Return: success.
5188 */
ufshcd_slave_alloc(struct scsi_device * sdev)5189 static int ufshcd_slave_alloc(struct scsi_device *sdev)
5190 {
5191 struct ufs_hba *hba;
5192
5193 hba = shost_priv(sdev->host);
5194
5195 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
5196 sdev->use_10_for_ms = 1;
5197
5198 /* DBD field should be set to 1 in mode sense(10) */
5199 sdev->set_dbd_for_ms = 1;
5200
5201 /* allow SCSI layer to restart the device in case of errors */
5202 sdev->allow_restart = 1;
5203
5204 /* REPORT SUPPORTED OPERATION CODES is not supported */
5205 sdev->no_report_opcodes = 1;
5206
5207 /* WRITE_SAME command is not supported */
5208 sdev->no_write_same = 1;
5209
5210 ufshcd_lu_init(hba, sdev);
5211
5212 ufshcd_setup_links(hba, sdev);
5213
5214 return 0;
5215 }
5216
5217 /**
5218 * ufshcd_change_queue_depth - change queue depth
5219 * @sdev: pointer to SCSI device
5220 * @depth: required depth to set
5221 *
5222 * Change queue depth and make sure the max. limits are not crossed.
5223 *
5224 * Return: new queue depth.
5225 */
ufshcd_change_queue_depth(struct scsi_device * sdev,int depth)5226 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
5227 {
5228 return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue));
5229 }
5230
5231 /**
5232 * ufshcd_device_configure - adjust SCSI device configurations
5233 * @sdev: pointer to SCSI device
5234 * @lim: queue limits
5235 *
5236 * Return: 0 (success).
5237 */
ufshcd_device_configure(struct scsi_device * sdev,struct queue_limits * lim)5238 static int ufshcd_device_configure(struct scsi_device *sdev,
5239 struct queue_limits *lim)
5240 {
5241 struct ufs_hba *hba = shost_priv(sdev->host);
5242 struct request_queue *q = sdev->request_queue;
5243
5244 lim->dma_pad_mask = PRDT_DATA_BYTE_COUNT_PAD - 1;
5245
5246 /*
5247 * Block runtime-pm until all consumers are added.
5248 * Refer ufshcd_setup_links().
5249 */
5250 if (is_device_wlun(sdev))
5251 pm_runtime_get_noresume(&sdev->sdev_gendev);
5252 else if (ufshcd_is_rpm_autosuspend_allowed(hba))
5253 sdev->rpm_autosuspend = 1;
5254 /*
5255 * Do not print messages during runtime PM to avoid never-ending cycles
5256 * of messages written back to storage by user space causing runtime
5257 * resume, causing more messages and so on.
5258 */
5259 sdev->silence_suspend = 1;
5260
5261 ufshcd_crypto_register(hba, q);
5262
5263 return 0;
5264 }
5265
5266 /**
5267 * ufshcd_slave_destroy - remove SCSI device configurations
5268 * @sdev: pointer to SCSI device
5269 */
ufshcd_slave_destroy(struct scsi_device * sdev)5270 static void ufshcd_slave_destroy(struct scsi_device *sdev)
5271 {
5272 struct ufs_hba *hba;
5273 unsigned long flags;
5274
5275 hba = shost_priv(sdev->host);
5276
5277 /* Drop the reference as it won't be needed anymore */
5278 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
5279 spin_lock_irqsave(hba->host->host_lock, flags);
5280 hba->ufs_device_wlun = NULL;
5281 spin_unlock_irqrestore(hba->host->host_lock, flags);
5282 } else if (hba->ufs_device_wlun) {
5283 struct device *supplier = NULL;
5284
5285 /* Ensure UFS Device WLUN exists and does not disappear */
5286 spin_lock_irqsave(hba->host->host_lock, flags);
5287 if (hba->ufs_device_wlun) {
5288 supplier = &hba->ufs_device_wlun->sdev_gendev;
5289 get_device(supplier);
5290 }
5291 spin_unlock_irqrestore(hba->host->host_lock, flags);
5292
5293 if (supplier) {
5294 /*
5295 * If a LUN fails to probe (e.g. absent BOOT WLUN), the
5296 * device will not have been registered but can still
5297 * have a device link holding a reference to the device.
5298 */
5299 device_link_remove(&sdev->sdev_gendev, supplier);
5300 put_device(supplier);
5301 }
5302 }
5303 }
5304
5305 /**
5306 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
5307 * @lrbp: pointer to local reference block of completed command
5308 * @scsi_status: SCSI command status
5309 *
5310 * Return: value base on SCSI command status.
5311 */
5312 static inline int
ufshcd_scsi_cmd_status(struct ufshcd_lrb * lrbp,int scsi_status)5313 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
5314 {
5315 int result = 0;
5316
5317 switch (scsi_status) {
5318 case SAM_STAT_CHECK_CONDITION:
5319 ufshcd_copy_sense_data(lrbp);
5320 fallthrough;
5321 case SAM_STAT_GOOD:
5322 result |= DID_OK << 16 | scsi_status;
5323 break;
5324 case SAM_STAT_TASK_SET_FULL:
5325 case SAM_STAT_BUSY:
5326 case SAM_STAT_TASK_ABORTED:
5327 ufshcd_copy_sense_data(lrbp);
5328 result |= scsi_status;
5329 break;
5330 default:
5331 result |= DID_ERROR << 16;
5332 break;
5333 } /* end of switch */
5334
5335 return result;
5336 }
5337
5338 /**
5339 * ufshcd_transfer_rsp_status - Get overall status of the response
5340 * @hba: per adapter instance
5341 * @lrbp: pointer to local reference block of completed command
5342 * @cqe: pointer to the completion queue entry
5343 *
5344 * Return: result of the command to notify SCSI midlayer.
5345 */
5346 static inline int
ufshcd_transfer_rsp_status(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,struct cq_entry * cqe)5347 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
5348 struct cq_entry *cqe)
5349 {
5350 int result = 0;
5351 int scsi_status;
5352 enum utp_ocs ocs;
5353 u8 upiu_flags;
5354 u32 resid;
5355
5356 upiu_flags = lrbp->ucd_rsp_ptr->header.flags;
5357 resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count);
5358 /*
5359 * Test !overflow instead of underflow to support UFS devices that do
5360 * not set either flag.
5361 */
5362 if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW))
5363 scsi_set_resid(lrbp->cmd, resid);
5364
5365 /* overall command status of utrd */
5366 ocs = ufshcd_get_tr_ocs(lrbp, cqe);
5367
5368 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5369 if (lrbp->ucd_rsp_ptr->header.response ||
5370 lrbp->ucd_rsp_ptr->header.status)
5371 ocs = OCS_SUCCESS;
5372 }
5373
5374 switch (ocs) {
5375 case OCS_SUCCESS:
5376 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5377 switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) {
5378 case UPIU_TRANSACTION_RESPONSE:
5379 /*
5380 * get the result based on SCSI status response
5381 * to notify the SCSI midlayer of the command status
5382 */
5383 scsi_status = lrbp->ucd_rsp_ptr->header.status;
5384 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
5385
5386 /*
5387 * Currently we are only supporting BKOPs exception
5388 * events hence we can ignore BKOPs exception event
5389 * during power management callbacks. BKOPs exception
5390 * event is not expected to be raised in runtime suspend
5391 * callback as it allows the urgent bkops.
5392 * During system suspend, we are anyway forcefully
5393 * disabling the bkops and if urgent bkops is needed
5394 * it will be enabled on system resume. Long term
5395 * solution could be to abort the system suspend if
5396 * UFS device needs urgent BKOPs.
5397 */
5398 if (!hba->pm_op_in_progress &&
5399 !ufshcd_eh_in_progress(hba) &&
5400 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5401 /* Flushed in suspend */
5402 schedule_work(&hba->eeh_work);
5403 break;
5404 case UPIU_TRANSACTION_REJECT_UPIU:
5405 /* TODO: handle Reject UPIU Response */
5406 result = DID_ERROR << 16;
5407 dev_err(hba->dev,
5408 "Reject UPIU not fully implemented\n");
5409 break;
5410 default:
5411 dev_err(hba->dev,
5412 "Unexpected request response code = %x\n",
5413 result);
5414 result = DID_ERROR << 16;
5415 break;
5416 }
5417 break;
5418 case OCS_ABORTED:
5419 case OCS_INVALID_COMMAND_STATUS:
5420 result |= DID_REQUEUE << 16;
5421 dev_warn(hba->dev,
5422 "OCS %s from controller for tag %d\n",
5423 (ocs == OCS_ABORTED ? "aborted" : "invalid"),
5424 lrbp->task_tag);
5425 break;
5426 case OCS_INVALID_CMD_TABLE_ATTR:
5427 case OCS_INVALID_PRDT_ATTR:
5428 case OCS_MISMATCH_DATA_BUF_SIZE:
5429 case OCS_MISMATCH_RESP_UPIU_SIZE:
5430 case OCS_PEER_COMM_FAILURE:
5431 case OCS_FATAL_ERROR:
5432 case OCS_DEVICE_FATAL_ERROR:
5433 case OCS_INVALID_CRYPTO_CONFIG:
5434 case OCS_GENERAL_CRYPTO_ERROR:
5435 default:
5436 result |= DID_ERROR << 16;
5437 dev_err(hba->dev,
5438 "OCS error from controller = %x for tag %d\n",
5439 ocs, lrbp->task_tag);
5440 ufshcd_print_evt_hist(hba);
5441 ufshcd_print_host_state(hba);
5442 break;
5443 } /* end of switch */
5444
5445 if ((host_byte(result) != DID_OK) &&
5446 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5447 ufshcd_print_tr(hba, lrbp->task_tag, true);
5448 return result;
5449 }
5450
ufshcd_is_auto_hibern8_error(struct ufs_hba * hba,u32 intr_mask)5451 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5452 u32 intr_mask)
5453 {
5454 if (!ufshcd_is_auto_hibern8_supported(hba) ||
5455 !ufshcd_is_auto_hibern8_enabled(hba))
5456 return false;
5457
5458 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5459 return false;
5460
5461 if (hba->active_uic_cmd &&
5462 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5463 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5464 return false;
5465
5466 return true;
5467 }
5468
5469 /**
5470 * ufshcd_uic_cmd_compl - handle completion of uic command
5471 * @hba: per adapter instance
5472 * @intr_status: interrupt status generated by the controller
5473 *
5474 * Return:
5475 * IRQ_HANDLED - If interrupt is valid
5476 * IRQ_NONE - If invalid interrupt
5477 */
ufshcd_uic_cmd_compl(struct ufs_hba * hba,u32 intr_status)5478 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
5479 {
5480 irqreturn_t retval = IRQ_NONE;
5481
5482 spin_lock(hba->host->host_lock);
5483 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5484 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5485
5486 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
5487 hba->active_uic_cmd->argument2 |=
5488 ufshcd_get_uic_cmd_result(hba);
5489 hba->active_uic_cmd->argument3 =
5490 ufshcd_get_dme_attr_val(hba);
5491 if (!hba->uic_async_done)
5492 hba->active_uic_cmd->cmd_active = 0;
5493 complete(&hba->active_uic_cmd->done);
5494 retval = IRQ_HANDLED;
5495 }
5496
5497 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
5498 hba->active_uic_cmd->cmd_active = 0;
5499 complete(hba->uic_async_done);
5500 retval = IRQ_HANDLED;
5501 }
5502
5503 if (retval == IRQ_HANDLED)
5504 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
5505 UFS_CMD_COMP);
5506 spin_unlock(hba->host->host_lock);
5507 return retval;
5508 }
5509
5510 /* Release the resources allocated for processing a SCSI command. */
ufshcd_release_scsi_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)5511 void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
5512 struct ufshcd_lrb *lrbp)
5513 {
5514 struct scsi_cmnd *cmd = lrbp->cmd;
5515
5516 scsi_dma_unmap(cmd);
5517 ufshcd_crypto_clear_prdt(hba, lrbp);
5518 ufshcd_release(hba);
5519 ufshcd_clk_scaling_update_busy(hba);
5520 }
5521
5522 /**
5523 * ufshcd_compl_one_cqe - handle a completion queue entry
5524 * @hba: per adapter instance
5525 * @task_tag: the task tag of the request to be completed
5526 * @cqe: pointer to the completion queue entry
5527 */
ufshcd_compl_one_cqe(struct ufs_hba * hba,int task_tag,struct cq_entry * cqe)5528 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
5529 struct cq_entry *cqe)
5530 {
5531 struct ufshcd_lrb *lrbp;
5532 struct scsi_cmnd *cmd;
5533 enum utp_ocs ocs;
5534
5535 lrbp = &hba->lrb[task_tag];
5536 lrbp->compl_time_stamp = ktime_get();
5537 cmd = lrbp->cmd;
5538 if (cmd) {
5539 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5540 ufshcd_update_monitor(hba, lrbp);
5541 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP);
5542 cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe);
5543 ufshcd_release_scsi_cmd(hba, lrbp);
5544 /* Do not touch lrbp after scsi done */
5545 scsi_done(cmd);
5546 } else if (hba->dev_cmd.complete) {
5547 if (cqe) {
5548 ocs = le32_to_cpu(cqe->status) & MASK_OCS;
5549 lrbp->utr_descriptor_ptr->header.ocs = ocs;
5550 }
5551 complete(hba->dev_cmd.complete);
5552 }
5553 }
5554
5555 /**
5556 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5557 * @hba: per adapter instance
5558 * @completed_reqs: bitmask that indicates which requests to complete
5559 */
__ufshcd_transfer_req_compl(struct ufs_hba * hba,unsigned long completed_reqs)5560 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5561 unsigned long completed_reqs)
5562 {
5563 int tag;
5564
5565 for_each_set_bit(tag, &completed_reqs, hba->nutrs)
5566 ufshcd_compl_one_cqe(hba, tag, NULL);
5567 }
5568
5569 /* Any value that is not an existing queue number is fine for this constant. */
5570 enum {
5571 UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1
5572 };
5573
ufshcd_clear_polled(struct ufs_hba * hba,unsigned long * completed_reqs)5574 static void ufshcd_clear_polled(struct ufs_hba *hba,
5575 unsigned long *completed_reqs)
5576 {
5577 int tag;
5578
5579 for_each_set_bit(tag, completed_reqs, hba->nutrs) {
5580 struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
5581
5582 if (!cmd)
5583 continue;
5584 if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED)
5585 __clear_bit(tag, completed_reqs);
5586 }
5587 }
5588
5589 /*
5590 * Return: > 0 if one or more commands have been completed or 0 if no
5591 * requests have been completed.
5592 */
ufshcd_poll(struct Scsi_Host * shost,unsigned int queue_num)5593 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
5594 {
5595 struct ufs_hba *hba = shost_priv(shost);
5596 unsigned long completed_reqs, flags;
5597 u32 tr_doorbell;
5598 struct ufs_hw_queue *hwq;
5599
5600 if (hba->mcq_enabled) {
5601 hwq = &hba->uhq[queue_num];
5602
5603 return ufshcd_mcq_poll_cqe_lock(hba, hwq);
5604 }
5605
5606 spin_lock_irqsave(&hba->outstanding_lock, flags);
5607 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5608 completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
5609 WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
5610 "completed: %#lx; outstanding: %#lx\n", completed_reqs,
5611 hba->outstanding_reqs);
5612 if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) {
5613 /* Do not complete polled requests from interrupt context. */
5614 ufshcd_clear_polled(hba, &completed_reqs);
5615 }
5616 hba->outstanding_reqs &= ~completed_reqs;
5617 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5618
5619 if (completed_reqs)
5620 __ufshcd_transfer_req_compl(hba, completed_reqs);
5621
5622 return completed_reqs != 0;
5623 }
5624
5625 /**
5626 * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is
5627 * invoked from the error handler context or ufshcd_host_reset_and_restore()
5628 * to complete the pending transfers and free the resources associated with
5629 * the scsi command.
5630 *
5631 * @hba: per adapter instance
5632 * @force_compl: This flag is set to true when invoked
5633 * from ufshcd_host_reset_and_restore() in which case it requires special
5634 * handling because the host controller has been reset by ufshcd_hba_stop().
5635 */
ufshcd_mcq_compl_pending_transfer(struct ufs_hba * hba,bool force_compl)5636 static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba,
5637 bool force_compl)
5638 {
5639 struct ufs_hw_queue *hwq;
5640 struct ufshcd_lrb *lrbp;
5641 struct scsi_cmnd *cmd;
5642 unsigned long flags;
5643 int tag;
5644
5645 for (tag = 0; tag < hba->nutrs; tag++) {
5646 lrbp = &hba->lrb[tag];
5647 cmd = lrbp->cmd;
5648 if (!ufshcd_cmd_inflight(cmd) ||
5649 test_bit(SCMD_STATE_COMPLETE, &cmd->state))
5650 continue;
5651
5652 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
5653
5654 if (force_compl) {
5655 ufshcd_mcq_compl_all_cqes_lock(hba, hwq);
5656 /*
5657 * For those cmds of which the cqes are not present
5658 * in the cq, complete them explicitly.
5659 */
5660 spin_lock_irqsave(&hwq->cq_lock, flags);
5661 if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
5662 set_host_byte(cmd, DID_REQUEUE);
5663 ufshcd_release_scsi_cmd(hba, lrbp);
5664 scsi_done(cmd);
5665 }
5666 spin_unlock_irqrestore(&hwq->cq_lock, flags);
5667 } else {
5668 ufshcd_mcq_poll_cqe_lock(hba, hwq);
5669 }
5670 }
5671 }
5672
5673 /**
5674 * ufshcd_transfer_req_compl - handle SCSI and query command completion
5675 * @hba: per adapter instance
5676 *
5677 * Return:
5678 * IRQ_HANDLED - If interrupt is valid
5679 * IRQ_NONE - If invalid interrupt
5680 */
ufshcd_transfer_req_compl(struct ufs_hba * hba)5681 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
5682 {
5683 /* Resetting interrupt aggregation counters first and reading the
5684 * DOOR_BELL afterward allows us to handle all the completed requests.
5685 * In order to prevent other interrupts starvation the DB is read once
5686 * after reset. The down side of this solution is the possibility of
5687 * false interrupt if device completes another request after resetting
5688 * aggregation and before reading the DB.
5689 */
5690 if (ufshcd_is_intr_aggr_allowed(hba) &&
5691 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5692 ufshcd_reset_intr_aggr(hba);
5693
5694 if (ufs_fail_completion(hba))
5695 return IRQ_HANDLED;
5696
5697 /*
5698 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
5699 * do not want polling to trigger spurious interrupt complaints.
5700 */
5701 ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
5702
5703 return IRQ_HANDLED;
5704 }
5705
__ufshcd_write_ee_control(struct ufs_hba * hba,u32 ee_ctrl_mask)5706 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
5707 {
5708 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5709 QUERY_ATTR_IDN_EE_CONTROL, 0, 0,
5710 &ee_ctrl_mask);
5711 }
5712
ufshcd_write_ee_control(struct ufs_hba * hba)5713 int ufshcd_write_ee_control(struct ufs_hba *hba)
5714 {
5715 int err;
5716
5717 mutex_lock(&hba->ee_ctrl_mutex);
5718 err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5719 mutex_unlock(&hba->ee_ctrl_mutex);
5720 if (err)
5721 dev_err(hba->dev, "%s: failed to write ee control %d\n",
5722 __func__, err);
5723 return err;
5724 }
5725
ufshcd_update_ee_control(struct ufs_hba * hba,u16 * mask,const u16 * other_mask,u16 set,u16 clr)5726 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask,
5727 const u16 *other_mask, u16 set, u16 clr)
5728 {
5729 u16 new_mask, ee_ctrl_mask;
5730 int err = 0;
5731
5732 mutex_lock(&hba->ee_ctrl_mutex);
5733 new_mask = (*mask & ~clr) | set;
5734 ee_ctrl_mask = new_mask | *other_mask;
5735 if (ee_ctrl_mask != hba->ee_ctrl_mask)
5736 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask);
5737 /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */
5738 if (!err) {
5739 hba->ee_ctrl_mask = ee_ctrl_mask;
5740 *mask = new_mask;
5741 }
5742 mutex_unlock(&hba->ee_ctrl_mutex);
5743 return err;
5744 }
5745
5746 /**
5747 * ufshcd_disable_ee - disable exception event
5748 * @hba: per-adapter instance
5749 * @mask: exception event to disable
5750 *
5751 * Disables exception event in the device so that the EVENT_ALERT
5752 * bit is not set.
5753 *
5754 * Return: zero on success, non-zero error value on failure.
5755 */
ufshcd_disable_ee(struct ufs_hba * hba,u16 mask)5756 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5757 {
5758 return ufshcd_update_ee_drv_mask(hba, 0, mask);
5759 }
5760
5761 /**
5762 * ufshcd_enable_ee - enable exception event
5763 * @hba: per-adapter instance
5764 * @mask: exception event to enable
5765 *
5766 * Enable corresponding exception event in the device to allow
5767 * device to alert host in critical scenarios.
5768 *
5769 * Return: zero on success, non-zero error value on failure.
5770 */
ufshcd_enable_ee(struct ufs_hba * hba,u16 mask)5771 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5772 {
5773 return ufshcd_update_ee_drv_mask(hba, mask, 0);
5774 }
5775
5776 /**
5777 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5778 * @hba: per-adapter instance
5779 *
5780 * Allow device to manage background operations on its own. Enabling
5781 * this might lead to inconsistent latencies during normal data transfers
5782 * as the device is allowed to manage its own way of handling background
5783 * operations.
5784 *
5785 * Return: zero on success, non-zero on failure.
5786 */
ufshcd_enable_auto_bkops(struct ufs_hba * hba)5787 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5788 {
5789 int err = 0;
5790
5791 if (hba->auto_bkops_enabled)
5792 goto out;
5793
5794 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5795 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5796 if (err) {
5797 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5798 __func__, err);
5799 goto out;
5800 }
5801
5802 hba->auto_bkops_enabled = true;
5803 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5804
5805 /* No need of URGENT_BKOPS exception from the device */
5806 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5807 if (err)
5808 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5809 __func__, err);
5810 out:
5811 return err;
5812 }
5813
5814 /**
5815 * ufshcd_disable_auto_bkops - block device in doing background operations
5816 * @hba: per-adapter instance
5817 *
5818 * Disabling background operations improves command response latency but
5819 * has drawback of device moving into critical state where the device is
5820 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5821 * host is idle so that BKOPS are managed effectively without any negative
5822 * impacts.
5823 *
5824 * Return: zero on success, non-zero on failure.
5825 */
ufshcd_disable_auto_bkops(struct ufs_hba * hba)5826 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5827 {
5828 int err = 0;
5829
5830 if (!hba->auto_bkops_enabled)
5831 goto out;
5832
5833 /*
5834 * If host assisted BKOPs is to be enabled, make sure
5835 * urgent bkops exception is allowed.
5836 */
5837 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5838 if (err) {
5839 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5840 __func__, err);
5841 goto out;
5842 }
5843
5844 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5845 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5846 if (err) {
5847 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5848 __func__, err);
5849 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5850 goto out;
5851 }
5852
5853 hba->auto_bkops_enabled = false;
5854 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5855 hba->is_urgent_bkops_lvl_checked = false;
5856 out:
5857 return err;
5858 }
5859
5860 /**
5861 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5862 * @hba: per adapter instance
5863 *
5864 * After a device reset the device may toggle the BKOPS_EN flag
5865 * to default value. The s/w tracking variables should be updated
5866 * as well. This function would change the auto-bkops state based on
5867 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5868 */
ufshcd_force_reset_auto_bkops(struct ufs_hba * hba)5869 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5870 {
5871 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5872 hba->auto_bkops_enabled = false;
5873 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5874 ufshcd_enable_auto_bkops(hba);
5875 } else {
5876 hba->auto_bkops_enabled = true;
5877 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5878 ufshcd_disable_auto_bkops(hba);
5879 }
5880 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5881 hba->is_urgent_bkops_lvl_checked = false;
5882 }
5883
ufshcd_get_bkops_status(struct ufs_hba * hba,u32 * status)5884 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5885 {
5886 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5887 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5888 }
5889
5890 /**
5891 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5892 * @hba: per-adapter instance
5893 *
5894 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5895 * flag in the device to permit background operations if the device
5896 * bkops_status is greater than or equal to the "hba->urgent_bkops_lvl",
5897 * disable otherwise.
5898 *
5899 * Return: 0 for success, non-zero in case of failure.
5900 *
5901 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5902 * to know whether auto bkops is enabled or disabled after this function
5903 * returns control to it.
5904 */
ufshcd_bkops_ctrl(struct ufs_hba * hba)5905 static int ufshcd_bkops_ctrl(struct ufs_hba *hba)
5906 {
5907 enum bkops_status status = hba->urgent_bkops_lvl;
5908 u32 curr_status = 0;
5909 int err;
5910
5911 err = ufshcd_get_bkops_status(hba, &curr_status);
5912 if (err) {
5913 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5914 __func__, err);
5915 goto out;
5916 } else if (curr_status > BKOPS_STATUS_MAX) {
5917 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5918 __func__, curr_status);
5919 err = -EINVAL;
5920 goto out;
5921 }
5922
5923 if (curr_status >= status)
5924 err = ufshcd_enable_auto_bkops(hba);
5925 else
5926 err = ufshcd_disable_auto_bkops(hba);
5927 out:
5928 return err;
5929 }
5930
ufshcd_get_ee_status(struct ufs_hba * hba,u32 * status)5931 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5932 {
5933 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5934 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5935 }
5936
ufshcd_bkops_exception_event_handler(struct ufs_hba * hba)5937 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5938 {
5939 int err;
5940 u32 curr_status = 0;
5941
5942 if (hba->is_urgent_bkops_lvl_checked)
5943 goto enable_auto_bkops;
5944
5945 err = ufshcd_get_bkops_status(hba, &curr_status);
5946 if (err) {
5947 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5948 __func__, err);
5949 goto out;
5950 }
5951
5952 /*
5953 * We are seeing that some devices are raising the urgent bkops
5954 * exception events even when BKOPS status doesn't indicate performace
5955 * impacted or critical. Handle these device by determining their urgent
5956 * bkops status at runtime.
5957 */
5958 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5959 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5960 __func__, curr_status);
5961 /* update the current status as the urgent bkops level */
5962 hba->urgent_bkops_lvl = curr_status;
5963 hba->is_urgent_bkops_lvl_checked = true;
5964 }
5965
5966 enable_auto_bkops:
5967 err = ufshcd_enable_auto_bkops(hba);
5968 out:
5969 if (err < 0)
5970 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5971 __func__, err);
5972 }
5973
ufshcd_temp_exception_event_handler(struct ufs_hba * hba,u16 status)5974 static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status)
5975 {
5976 u32 value;
5977
5978 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5979 QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value))
5980 return;
5981
5982 dev_info(hba->dev, "exception Tcase %d\n", value - 80);
5983
5984 ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP);
5985
5986 /*
5987 * A placeholder for the platform vendors to add whatever additional
5988 * steps required
5989 */
5990 }
5991
__ufshcd_wb_toggle(struct ufs_hba * hba,bool set,enum flag_idn idn)5992 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn)
5993 {
5994 u8 index;
5995 enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG :
5996 UPIU_QUERY_OPCODE_CLEAR_FLAG;
5997
5998 index = ufshcd_wb_get_query_index(hba);
5999 return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL);
6000 }
6001
ufshcd_wb_toggle(struct ufs_hba * hba,bool enable)6002 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable)
6003 {
6004 int ret;
6005
6006 if (!ufshcd_is_wb_allowed(hba) ||
6007 hba->dev_info.wb_enabled == enable)
6008 return 0;
6009
6010 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN);
6011 if (ret) {
6012 dev_err(hba->dev, "%s: Write Booster %s failed %d\n",
6013 __func__, enable ? "enabling" : "disabling", ret);
6014 return ret;
6015 }
6016
6017 hba->dev_info.wb_enabled = enable;
6018 dev_dbg(hba->dev, "%s: Write Booster %s\n",
6019 __func__, enable ? "enabled" : "disabled");
6020
6021 return ret;
6022 }
6023
ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba * hba,bool enable)6024 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
6025 bool enable)
6026 {
6027 int ret;
6028
6029 ret = __ufshcd_wb_toggle(hba, enable,
6030 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8);
6031 if (ret) {
6032 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n",
6033 __func__, enable ? "enabling" : "disabling", ret);
6034 return;
6035 }
6036 dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n",
6037 __func__, enable ? "enabled" : "disabled");
6038 }
6039
ufshcd_wb_toggle_buf_flush(struct ufs_hba * hba,bool enable)6040 int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable)
6041 {
6042 int ret;
6043
6044 if (!ufshcd_is_wb_allowed(hba) ||
6045 hba->dev_info.wb_buf_flush_enabled == enable)
6046 return 0;
6047
6048 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN);
6049 if (ret) {
6050 dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n",
6051 __func__, enable ? "enabling" : "disabling", ret);
6052 return ret;
6053 }
6054
6055 hba->dev_info.wb_buf_flush_enabled = enable;
6056 dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n",
6057 __func__, enable ? "enabled" : "disabled");
6058
6059 return ret;
6060 }
6061
ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba * hba,u32 avail_buf)6062 static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
6063 u32 avail_buf)
6064 {
6065 u32 cur_buf;
6066 int ret;
6067 u8 index;
6068
6069 index = ufshcd_wb_get_query_index(hba);
6070 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6071 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
6072 index, 0, &cur_buf);
6073 if (ret) {
6074 dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n",
6075 __func__, ret);
6076 return false;
6077 }
6078
6079 if (!cur_buf) {
6080 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
6081 cur_buf);
6082 return false;
6083 }
6084 /* Let it continue to flush when available buffer exceeds threshold */
6085 return avail_buf < hba->vps->wb_flush_threshold;
6086 }
6087
ufshcd_wb_force_disable(struct ufs_hba * hba)6088 static void ufshcd_wb_force_disable(struct ufs_hba *hba)
6089 {
6090 if (ufshcd_is_wb_buf_flush_allowed(hba))
6091 ufshcd_wb_toggle_buf_flush(hba, false);
6092
6093 ufshcd_wb_toggle_buf_flush_during_h8(hba, false);
6094 ufshcd_wb_toggle(hba, false);
6095 hba->caps &= ~UFSHCD_CAP_WB_EN;
6096
6097 dev_info(hba->dev, "%s: WB force disabled\n", __func__);
6098 }
6099
ufshcd_is_wb_buf_lifetime_available(struct ufs_hba * hba)6100 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba)
6101 {
6102 u32 lifetime;
6103 int ret;
6104 u8 index;
6105
6106 index = ufshcd_wb_get_query_index(hba);
6107 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6108 QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST,
6109 index, 0, &lifetime);
6110 if (ret) {
6111 dev_err(hba->dev,
6112 "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n",
6113 __func__, ret);
6114 return false;
6115 }
6116
6117 if (lifetime == UFS_WB_EXCEED_LIFETIME) {
6118 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n",
6119 __func__, lifetime);
6120 return false;
6121 }
6122
6123 dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n",
6124 __func__, lifetime);
6125
6126 return true;
6127 }
6128
ufshcd_wb_need_flush(struct ufs_hba * hba)6129 static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
6130 {
6131 int ret;
6132 u32 avail_buf;
6133 u8 index;
6134
6135 if (!ufshcd_is_wb_allowed(hba))
6136 return false;
6137
6138 if (!ufshcd_is_wb_buf_lifetime_available(hba)) {
6139 ufshcd_wb_force_disable(hba);
6140 return false;
6141 }
6142
6143 /*
6144 * The ufs device needs the vcc to be ON to flush.
6145 * With user-space reduction enabled, it's enough to enable flush
6146 * by checking only the available buffer. The threshold
6147 * defined here is > 90% full.
6148 * With user-space preserved enabled, the current-buffer
6149 * should be checked too because the wb buffer size can reduce
6150 * when disk tends to be full. This info is provided by current
6151 * buffer (dCurrentWriteBoosterBufferSize). There's no point in
6152 * keeping vcc on when current buffer is empty.
6153 */
6154 index = ufshcd_wb_get_query_index(hba);
6155 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6156 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
6157 index, 0, &avail_buf);
6158 if (ret) {
6159 dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n",
6160 __func__, ret);
6161 return false;
6162 }
6163
6164 if (!hba->dev_info.b_presrv_uspc_en)
6165 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10);
6166
6167 return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
6168 }
6169
ufshcd_rpm_dev_flush_recheck_work(struct work_struct * work)6170 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
6171 {
6172 struct ufs_hba *hba = container_of(to_delayed_work(work),
6173 struct ufs_hba,
6174 rpm_dev_flush_recheck_work);
6175 /*
6176 * To prevent unnecessary VCC power drain after device finishes
6177 * WriteBooster buffer flush or Auto BKOPs, force runtime resume
6178 * after a certain delay to recheck the threshold by next runtime
6179 * suspend.
6180 */
6181 ufshcd_rpm_get_sync(hba);
6182 ufshcd_rpm_put_sync(hba);
6183 }
6184
6185 /**
6186 * ufshcd_exception_event_handler - handle exceptions raised by device
6187 * @work: pointer to work data
6188 *
6189 * Read bExceptionEventStatus attribute from the device and handle the
6190 * exception event accordingly.
6191 */
ufshcd_exception_event_handler(struct work_struct * work)6192 static void ufshcd_exception_event_handler(struct work_struct *work)
6193 {
6194 struct ufs_hba *hba;
6195 int err;
6196 u32 status = 0;
6197 hba = container_of(work, struct ufs_hba, eeh_work);
6198
6199 ufshcd_scsi_block_requests(hba);
6200 err = ufshcd_get_ee_status(hba, &status);
6201 if (err) {
6202 dev_err(hba->dev, "%s: failed to get exception status %d\n",
6203 __func__, err);
6204 goto out;
6205 }
6206
6207 trace_ufshcd_exception_event(dev_name(hba->dev), status);
6208
6209 if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS)
6210 ufshcd_bkops_exception_event_handler(hba);
6211
6212 if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP)
6213 ufshcd_temp_exception_event_handler(hba, status);
6214
6215 ufs_debugfs_exception_event(hba, status);
6216 out:
6217 ufshcd_scsi_unblock_requests(hba);
6218 }
6219
6220 /* Complete requests that have door-bell cleared */
ufshcd_complete_requests(struct ufs_hba * hba,bool force_compl)6221 static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl)
6222 {
6223 if (hba->mcq_enabled)
6224 ufshcd_mcq_compl_pending_transfer(hba, force_compl);
6225 else
6226 ufshcd_transfer_req_compl(hba);
6227
6228 ufshcd_tmc_handler(hba);
6229 }
6230
6231 /**
6232 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
6233 * to recover from the DL NAC errors or not.
6234 * @hba: per-adapter instance
6235 *
6236 * Return: true if error handling is required, false otherwise.
6237 */
ufshcd_quirk_dl_nac_errors(struct ufs_hba * hba)6238 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
6239 {
6240 unsigned long flags;
6241 bool err_handling = true;
6242
6243 spin_lock_irqsave(hba->host->host_lock, flags);
6244 /*
6245 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
6246 * device fatal error and/or DL NAC & REPLAY timeout errors.
6247 */
6248 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
6249 goto out;
6250
6251 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
6252 ((hba->saved_err & UIC_ERROR) &&
6253 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
6254 goto out;
6255
6256 if ((hba->saved_err & UIC_ERROR) &&
6257 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
6258 int err;
6259 /*
6260 * wait for 50ms to see if we can get any other errors or not.
6261 */
6262 spin_unlock_irqrestore(hba->host->host_lock, flags);
6263 msleep(50);
6264 spin_lock_irqsave(hba->host->host_lock, flags);
6265
6266 /*
6267 * now check if we have got any other severe errors other than
6268 * DL NAC error?
6269 */
6270 if ((hba->saved_err & INT_FATAL_ERRORS) ||
6271 ((hba->saved_err & UIC_ERROR) &&
6272 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
6273 goto out;
6274
6275 /*
6276 * As DL NAC is the only error received so far, send out NOP
6277 * command to confirm if link is still active or not.
6278 * - If we don't get any response then do error recovery.
6279 * - If we get response then clear the DL NAC error bit.
6280 */
6281
6282 spin_unlock_irqrestore(hba->host->host_lock, flags);
6283 err = ufshcd_verify_dev_init(hba);
6284 spin_lock_irqsave(hba->host->host_lock, flags);
6285
6286 if (err)
6287 goto out;
6288
6289 /* Link seems to be alive hence ignore the DL NAC errors */
6290 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
6291 hba->saved_err &= ~UIC_ERROR;
6292 /* clear NAC error */
6293 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6294 if (!hba->saved_uic_err)
6295 err_handling = false;
6296 }
6297 out:
6298 spin_unlock_irqrestore(hba->host->host_lock, flags);
6299 return err_handling;
6300 }
6301
6302 /* host lock must be held before calling this func */
ufshcd_is_saved_err_fatal(struct ufs_hba * hba)6303 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
6304 {
6305 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
6306 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
6307 }
6308
ufshcd_schedule_eh_work(struct ufs_hba * hba)6309 void ufshcd_schedule_eh_work(struct ufs_hba *hba)
6310 {
6311 lockdep_assert_held(hba->host->host_lock);
6312
6313 /* handle fatal errors only when link is not in error state */
6314 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6315 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6316 ufshcd_is_saved_err_fatal(hba))
6317 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
6318 else
6319 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
6320 queue_work(hba->eh_wq, &hba->eh_work);
6321 }
6322 }
6323
ufshcd_force_error_recovery(struct ufs_hba * hba)6324 static void ufshcd_force_error_recovery(struct ufs_hba *hba)
6325 {
6326 spin_lock_irq(hba->host->host_lock);
6327 hba->force_reset = true;
6328 ufshcd_schedule_eh_work(hba);
6329 spin_unlock_irq(hba->host->host_lock);
6330 }
6331
ufshcd_clk_scaling_allow(struct ufs_hba * hba,bool allow)6332 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
6333 {
6334 mutex_lock(&hba->wb_mutex);
6335 down_write(&hba->clk_scaling_lock);
6336 hba->clk_scaling.is_allowed = allow;
6337 up_write(&hba->clk_scaling_lock);
6338 mutex_unlock(&hba->wb_mutex);
6339 }
6340
ufshcd_clk_scaling_suspend(struct ufs_hba * hba,bool suspend)6341 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
6342 {
6343 if (suspend) {
6344 if (hba->clk_scaling.is_enabled)
6345 ufshcd_suspend_clkscaling(hba);
6346 ufshcd_clk_scaling_allow(hba, false);
6347 } else {
6348 ufshcd_clk_scaling_allow(hba, true);
6349 if (hba->clk_scaling.is_enabled)
6350 ufshcd_resume_clkscaling(hba);
6351 }
6352 }
6353
ufshcd_err_handling_prepare(struct ufs_hba * hba)6354 static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
6355 {
6356 ufshcd_rpm_get_sync(hba);
6357 if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) ||
6358 hba->is_sys_suspended) {
6359 enum ufs_pm_op pm_op;
6360
6361 /*
6362 * Don't assume anything of resume, if
6363 * resume fails, irq and clocks can be OFF, and powers
6364 * can be OFF or in LPM.
6365 */
6366 ufshcd_setup_hba_vreg(hba, true);
6367 ufshcd_enable_irq(hba);
6368 ufshcd_setup_vreg(hba, true);
6369 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
6370 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
6371 ufshcd_hold(hba);
6372 if (!ufshcd_is_clkgating_allowed(hba))
6373 ufshcd_setup_clocks(hba, true);
6374 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
6375 ufshcd_vops_resume(hba, pm_op);
6376 } else {
6377 ufshcd_hold(hba);
6378 if (ufshcd_is_clkscaling_supported(hba) &&
6379 hba->clk_scaling.is_enabled)
6380 ufshcd_suspend_clkscaling(hba);
6381 ufshcd_clk_scaling_allow(hba, false);
6382 }
6383 ufshcd_scsi_block_requests(hba);
6384 /* Wait for ongoing ufshcd_queuecommand() calls to finish. */
6385 blk_mq_wait_quiesce_done(&hba->host->tag_set);
6386 cancel_work_sync(&hba->eeh_work);
6387 }
6388
ufshcd_err_handling_unprepare(struct ufs_hba * hba)6389 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
6390 {
6391 ufshcd_scsi_unblock_requests(hba);
6392 ufshcd_release(hba);
6393 if (ufshcd_is_clkscaling_supported(hba))
6394 ufshcd_clk_scaling_suspend(hba, false);
6395 ufshcd_rpm_put(hba);
6396 }
6397
ufshcd_err_handling_should_stop(struct ufs_hba * hba)6398 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
6399 {
6400 return (!hba->is_powered || hba->shutting_down ||
6401 !hba->ufs_device_wlun ||
6402 hba->ufshcd_state == UFSHCD_STATE_ERROR ||
6403 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
6404 ufshcd_is_link_broken(hba))));
6405 }
6406
6407 #ifdef CONFIG_PM
ufshcd_recover_pm_error(struct ufs_hba * hba)6408 static void ufshcd_recover_pm_error(struct ufs_hba *hba)
6409 {
6410 struct Scsi_Host *shost = hba->host;
6411 struct scsi_device *sdev;
6412 struct request_queue *q;
6413 int ret;
6414
6415 hba->is_sys_suspended = false;
6416 /*
6417 * Set RPM status of wlun device to RPM_ACTIVE,
6418 * this also clears its runtime error.
6419 */
6420 ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev);
6421
6422 /* hba device might have a runtime error otherwise */
6423 if (ret)
6424 ret = pm_runtime_set_active(hba->dev);
6425 /*
6426 * If wlun device had runtime error, we also need to resume those
6427 * consumer scsi devices in case any of them has failed to be
6428 * resumed due to supplier runtime resume failure. This is to unblock
6429 * blk_queue_enter in case there are bios waiting inside it.
6430 */
6431 if (!ret) {
6432 shost_for_each_device(sdev, shost) {
6433 q = sdev->request_queue;
6434 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
6435 q->rpm_status == RPM_SUSPENDING))
6436 pm_request_resume(q->dev);
6437 }
6438 }
6439 }
6440 #else
ufshcd_recover_pm_error(struct ufs_hba * hba)6441 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
6442 {
6443 }
6444 #endif
6445
ufshcd_is_pwr_mode_restore_needed(struct ufs_hba * hba)6446 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
6447 {
6448 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
6449 u32 mode;
6450
6451 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
6452
6453 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
6454 return true;
6455
6456 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
6457 return true;
6458
6459 return false;
6460 }
6461
ufshcd_abort_one(struct request * rq,void * priv)6462 static bool ufshcd_abort_one(struct request *rq, void *priv)
6463 {
6464 int *ret = priv;
6465 u32 tag = rq->tag;
6466 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
6467 struct scsi_device *sdev = cmd->device;
6468 struct Scsi_Host *shost = sdev->host;
6469 struct ufs_hba *hba = shost_priv(shost);
6470
6471 *ret = ufshcd_try_to_abort_task(hba, tag);
6472 dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
6473 hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
6474 *ret ? "failed" : "succeeded");
6475
6476 return *ret == 0;
6477 }
6478
6479 /**
6480 * ufshcd_abort_all - Abort all pending commands.
6481 * @hba: Host bus adapter pointer.
6482 *
6483 * Return: true if and only if the host controller needs to be reset.
6484 */
ufshcd_abort_all(struct ufs_hba * hba)6485 static bool ufshcd_abort_all(struct ufs_hba *hba)
6486 {
6487 int tag, ret = 0;
6488
6489 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret);
6490 if (ret)
6491 goto out;
6492
6493 /* Clear pending task management requests */
6494 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6495 ret = ufshcd_clear_tm_cmd(hba, tag);
6496 if (ret)
6497 goto out;
6498 }
6499
6500 out:
6501 /* Complete the requests that are cleared by s/w */
6502 ufshcd_complete_requests(hba, false);
6503
6504 return ret != 0;
6505 }
6506
6507 /**
6508 * ufshcd_err_handler - handle UFS errors that require s/w attention
6509 * @work: pointer to work structure
6510 */
ufshcd_err_handler(struct work_struct * work)6511 static void ufshcd_err_handler(struct work_struct *work)
6512 {
6513 int retries = MAX_ERR_HANDLER_RETRIES;
6514 struct ufs_hba *hba;
6515 unsigned long flags;
6516 bool needs_restore;
6517 bool needs_reset;
6518 int pmc_err;
6519
6520 hba = container_of(work, struct ufs_hba, eh_work);
6521
6522 dev_info(hba->dev,
6523 "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n",
6524 __func__, ufshcd_state_name[hba->ufshcd_state],
6525 hba->is_powered, hba->shutting_down, hba->saved_err,
6526 hba->saved_uic_err, hba->force_reset,
6527 ufshcd_is_link_broken(hba) ? "; link is broken" : "");
6528
6529 down(&hba->host_sem);
6530 spin_lock_irqsave(hba->host->host_lock, flags);
6531 if (ufshcd_err_handling_should_stop(hba)) {
6532 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6533 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6534 spin_unlock_irqrestore(hba->host->host_lock, flags);
6535 up(&hba->host_sem);
6536 return;
6537 }
6538 ufshcd_set_eh_in_progress(hba);
6539 spin_unlock_irqrestore(hba->host->host_lock, flags);
6540 ufshcd_err_handling_prepare(hba);
6541 /* Complete requests that have door-bell cleared by h/w */
6542 ufshcd_complete_requests(hba, false);
6543 spin_lock_irqsave(hba->host->host_lock, flags);
6544 again:
6545 needs_restore = false;
6546 needs_reset = false;
6547
6548 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6549 hba->ufshcd_state = UFSHCD_STATE_RESET;
6550 /*
6551 * A full reset and restore might have happened after preparation
6552 * is finished, double check whether we should stop.
6553 */
6554 if (ufshcd_err_handling_should_stop(hba))
6555 goto skip_err_handling;
6556
6557 if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) &&
6558 !hba->force_reset) {
6559 bool ret;
6560
6561 spin_unlock_irqrestore(hba->host->host_lock, flags);
6562 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
6563 ret = ufshcd_quirk_dl_nac_errors(hba);
6564 spin_lock_irqsave(hba->host->host_lock, flags);
6565 if (!ret && ufshcd_err_handling_should_stop(hba))
6566 goto skip_err_handling;
6567 }
6568
6569 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6570 (hba->saved_uic_err &&
6571 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6572 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6573
6574 spin_unlock_irqrestore(hba->host->host_lock, flags);
6575 ufshcd_print_host_state(hba);
6576 ufshcd_print_pwr_info(hba);
6577 ufshcd_print_evt_hist(hba);
6578 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6579 ufshcd_print_trs_all(hba, pr_prdt);
6580 spin_lock_irqsave(hba->host->host_lock, flags);
6581 }
6582
6583 /*
6584 * if host reset is required then skip clearing the pending
6585 * transfers forcefully because they will get cleared during
6586 * host reset and restore
6587 */
6588 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6589 ufshcd_is_saved_err_fatal(hba) ||
6590 ((hba->saved_err & UIC_ERROR) &&
6591 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6592 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
6593 needs_reset = true;
6594 goto do_reset;
6595 }
6596
6597 /*
6598 * If LINERESET was caught, UFS might have been put to PWM mode,
6599 * check if power mode restore is needed.
6600 */
6601 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6602 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6603 if (!hba->saved_uic_err)
6604 hba->saved_err &= ~UIC_ERROR;
6605 spin_unlock_irqrestore(hba->host->host_lock, flags);
6606 if (ufshcd_is_pwr_mode_restore_needed(hba))
6607 needs_restore = true;
6608 spin_lock_irqsave(hba->host->host_lock, flags);
6609 if (!hba->saved_err && !needs_restore)
6610 goto skip_err_handling;
6611 }
6612
6613 hba->silence_err_logs = true;
6614 /* release lock as clear command might sleep */
6615 spin_unlock_irqrestore(hba->host->host_lock, flags);
6616
6617 needs_reset = ufshcd_abort_all(hba);
6618
6619 spin_lock_irqsave(hba->host->host_lock, flags);
6620 hba->silence_err_logs = false;
6621 if (needs_reset)
6622 goto do_reset;
6623
6624 /*
6625 * After all reqs and tasks are cleared from doorbell,
6626 * now it is safe to retore power mode.
6627 */
6628 if (needs_restore) {
6629 spin_unlock_irqrestore(hba->host->host_lock, flags);
6630 /*
6631 * Hold the scaling lock just in case dev cmds
6632 * are sent via bsg and/or sysfs.
6633 */
6634 down_write(&hba->clk_scaling_lock);
6635 hba->force_pmc = true;
6636 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6637 if (pmc_err) {
6638 needs_reset = true;
6639 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6640 __func__, pmc_err);
6641 }
6642 hba->force_pmc = false;
6643 ufshcd_print_pwr_info(hba);
6644 up_write(&hba->clk_scaling_lock);
6645 spin_lock_irqsave(hba->host->host_lock, flags);
6646 }
6647
6648 do_reset:
6649 /* Fatal errors need reset */
6650 if (needs_reset) {
6651 int err;
6652
6653 hba->force_reset = false;
6654 spin_unlock_irqrestore(hba->host->host_lock, flags);
6655 err = ufshcd_reset_and_restore(hba);
6656 if (err)
6657 dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6658 __func__, err);
6659 else
6660 ufshcd_recover_pm_error(hba);
6661 spin_lock_irqsave(hba->host->host_lock, flags);
6662 }
6663
6664 skip_err_handling:
6665 if (!needs_reset) {
6666 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6667 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6668 if (hba->saved_err || hba->saved_uic_err)
6669 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6670 __func__, hba->saved_err, hba->saved_uic_err);
6671 }
6672 /* Exit in an operational state or dead */
6673 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
6674 hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6675 if (--retries)
6676 goto again;
6677 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6678 }
6679 ufshcd_clear_eh_in_progress(hba);
6680 spin_unlock_irqrestore(hba->host->host_lock, flags);
6681 ufshcd_err_handling_unprepare(hba);
6682 up(&hba->host_sem);
6683
6684 dev_info(hba->dev, "%s finished; HBA state %s\n", __func__,
6685 ufshcd_state_name[hba->ufshcd_state]);
6686 }
6687
6688 /**
6689 * ufshcd_update_uic_error - check and set fatal UIC error flags.
6690 * @hba: per-adapter instance
6691 *
6692 * Return:
6693 * IRQ_HANDLED - If interrupt is valid
6694 * IRQ_NONE - If invalid interrupt
6695 */
ufshcd_update_uic_error(struct ufs_hba * hba)6696 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
6697 {
6698 u32 reg;
6699 irqreturn_t retval = IRQ_NONE;
6700
6701 /* PHY layer error */
6702 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
6703 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
6704 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
6705 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
6706 /*
6707 * To know whether this error is fatal or not, DB timeout
6708 * must be checked but this error is handled separately.
6709 */
6710 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6711 dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6712 __func__);
6713
6714 /* Got a LINERESET indication. */
6715 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6716 struct uic_command *cmd = NULL;
6717
6718 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6719 if (hba->uic_async_done && hba->active_uic_cmd)
6720 cmd = hba->active_uic_cmd;
6721 /*
6722 * Ignore the LINERESET during power mode change
6723 * operation via DME_SET command.
6724 */
6725 if (cmd && (cmd->command == UIC_CMD_DME_SET))
6726 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6727 }
6728 retval |= IRQ_HANDLED;
6729 }
6730
6731 /* PA_INIT_ERROR is fatal and needs UIC reset */
6732 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
6733 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6734 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
6735 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
6736
6737 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6738 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6739 else if (hba->dev_quirks &
6740 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6741 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6742 hba->uic_error |=
6743 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6744 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6745 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6746 }
6747 retval |= IRQ_HANDLED;
6748 }
6749
6750 /* UIC NL/TL/DME errors needs software retry */
6751 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
6752 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6753 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
6754 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
6755 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
6756 retval |= IRQ_HANDLED;
6757 }
6758
6759 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
6760 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6761 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
6762 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
6763 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
6764 retval |= IRQ_HANDLED;
6765 }
6766
6767 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
6768 if ((reg & UIC_DME_ERROR) &&
6769 (reg & UIC_DME_ERROR_CODE_MASK)) {
6770 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
6771 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6772 retval |= IRQ_HANDLED;
6773 }
6774
6775 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6776 __func__, hba->uic_error);
6777 return retval;
6778 }
6779
6780 /**
6781 * ufshcd_check_errors - Check for errors that need s/w attention
6782 * @hba: per-adapter instance
6783 * @intr_status: interrupt status generated by the controller
6784 *
6785 * Return:
6786 * IRQ_HANDLED - If interrupt is valid
6787 * IRQ_NONE - If invalid interrupt
6788 */
ufshcd_check_errors(struct ufs_hba * hba,u32 intr_status)6789 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
6790 {
6791 bool queue_eh_work = false;
6792 irqreturn_t retval = IRQ_NONE;
6793
6794 spin_lock(hba->host->host_lock);
6795 hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6796
6797 if (hba->errors & INT_FATAL_ERRORS) {
6798 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6799 hba->errors);
6800 queue_eh_work = true;
6801 }
6802
6803 if (hba->errors & UIC_ERROR) {
6804 hba->uic_error = 0;
6805 retval = ufshcd_update_uic_error(hba);
6806 if (hba->uic_error)
6807 queue_eh_work = true;
6808 }
6809
6810 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6811 dev_err(hba->dev,
6812 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6813 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6814 "Enter" : "Exit",
6815 hba->errors, ufshcd_get_upmcrs(hba));
6816 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
6817 hba->errors);
6818 ufshcd_set_link_broken(hba);
6819 queue_eh_work = true;
6820 }
6821
6822 if (queue_eh_work) {
6823 /*
6824 * update the transfer error masks to sticky bits, let's do this
6825 * irrespective of current ufshcd_state.
6826 */
6827 hba->saved_err |= hba->errors;
6828 hba->saved_uic_err |= hba->uic_error;
6829
6830 /* dump controller state before resetting */
6831 if ((hba->saved_err &
6832 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6833 (hba->saved_uic_err &&
6834 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6835 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6836 __func__, hba->saved_err,
6837 hba->saved_uic_err);
6838 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6839 "host_regs: ");
6840 ufshcd_print_pwr_info(hba);
6841 }
6842 ufshcd_schedule_eh_work(hba);
6843 retval |= IRQ_HANDLED;
6844 }
6845 /*
6846 * if (!queue_eh_work) -
6847 * Other errors are either non-fatal where host recovers
6848 * itself without s/w intervention or errors that will be
6849 * handled by the SCSI core layer.
6850 */
6851 hba->errors = 0;
6852 hba->uic_error = 0;
6853 spin_unlock(hba->host->host_lock);
6854 return retval;
6855 }
6856
6857 /**
6858 * ufshcd_tmc_handler - handle task management function completion
6859 * @hba: per adapter instance
6860 *
6861 * Return:
6862 * IRQ_HANDLED - If interrupt is valid
6863 * IRQ_NONE - If invalid interrupt
6864 */
ufshcd_tmc_handler(struct ufs_hba * hba)6865 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
6866 {
6867 unsigned long flags, pending, issued;
6868 irqreturn_t ret = IRQ_NONE;
6869 int tag;
6870
6871 spin_lock_irqsave(hba->host->host_lock, flags);
6872 pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
6873 issued = hba->outstanding_tasks & ~pending;
6874 for_each_set_bit(tag, &issued, hba->nutmrs) {
6875 struct request *req = hba->tmf_rqs[tag];
6876 struct completion *c = req->end_io_data;
6877
6878 complete(c);
6879 ret = IRQ_HANDLED;
6880 }
6881 spin_unlock_irqrestore(hba->host->host_lock, flags);
6882
6883 return ret;
6884 }
6885
6886 /**
6887 * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events
6888 * @hba: per adapter instance
6889 *
6890 * Return: IRQ_HANDLED if interrupt is handled.
6891 */
ufshcd_handle_mcq_cq_events(struct ufs_hba * hba)6892 static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba)
6893 {
6894 struct ufs_hw_queue *hwq;
6895 unsigned long outstanding_cqs;
6896 unsigned int nr_queues;
6897 int i, ret;
6898 u32 events;
6899
6900 ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs);
6901 if (ret)
6902 outstanding_cqs = (1U << hba->nr_hw_queues) - 1;
6903
6904 /* Exclude the poll queues */
6905 nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL];
6906 for_each_set_bit(i, &outstanding_cqs, nr_queues) {
6907 hwq = &hba->uhq[i];
6908
6909 events = ufshcd_mcq_read_cqis(hba, i);
6910 if (events)
6911 ufshcd_mcq_write_cqis(hba, events, i);
6912
6913 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
6914 ufshcd_mcq_poll_cqe_lock(hba, hwq);
6915 }
6916
6917 return IRQ_HANDLED;
6918 }
6919
6920 /**
6921 * ufshcd_sl_intr - Interrupt service routine
6922 * @hba: per adapter instance
6923 * @intr_status: contains interrupts generated by the controller
6924 *
6925 * Return:
6926 * IRQ_HANDLED - If interrupt is valid
6927 * IRQ_NONE - If invalid interrupt
6928 */
ufshcd_sl_intr(struct ufs_hba * hba,u32 intr_status)6929 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
6930 {
6931 irqreturn_t retval = IRQ_NONE;
6932
6933 if (intr_status & UFSHCD_UIC_MASK)
6934 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
6935
6936 if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6937 retval |= ufshcd_check_errors(hba, intr_status);
6938
6939 if (intr_status & UTP_TASK_REQ_COMPL)
6940 retval |= ufshcd_tmc_handler(hba);
6941
6942 if (intr_status & UTP_TRANSFER_REQ_COMPL)
6943 retval |= ufshcd_transfer_req_compl(hba);
6944
6945 if (intr_status & MCQ_CQ_EVENT_STATUS)
6946 retval |= ufshcd_handle_mcq_cq_events(hba);
6947
6948 return retval;
6949 }
6950
6951 /**
6952 * ufshcd_intr - Main interrupt service routine
6953 * @irq: irq number
6954 * @__hba: pointer to adapter instance
6955 *
6956 * Return:
6957 * IRQ_HANDLED - If interrupt is valid
6958 * IRQ_NONE - If invalid interrupt
6959 */
ufshcd_intr(int irq,void * __hba)6960 static irqreturn_t ufshcd_intr(int irq, void *__hba)
6961 {
6962 u32 intr_status, enabled_intr_status = 0;
6963 irqreturn_t retval = IRQ_NONE;
6964 struct ufs_hba *hba = __hba;
6965 int retries = hba->nutrs;
6966
6967 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6968 hba->ufs_stats.last_intr_status = intr_status;
6969 hba->ufs_stats.last_intr_ts = local_clock();
6970
6971 /*
6972 * There could be max of hba->nutrs reqs in flight and in worst case
6973 * if the reqs get finished 1 by 1 after the interrupt status is
6974 * read, make sure we handle them by checking the interrupt status
6975 * again in a loop until we process all of the reqs before returning.
6976 */
6977 while (intr_status && retries--) {
6978 enabled_intr_status =
6979 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6980 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
6981 if (enabled_intr_status)
6982 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
6983
6984 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6985 }
6986
6987 if (enabled_intr_status && retval == IRQ_NONE &&
6988 (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) ||
6989 hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) {
6990 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6991 __func__,
6992 intr_status,
6993 hba->ufs_stats.last_intr_status,
6994 enabled_intr_status);
6995 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6996 }
6997
6998 return retval;
6999 }
7000
ufshcd_clear_tm_cmd(struct ufs_hba * hba,int tag)7001 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
7002 {
7003 int err = 0;
7004 u32 mask = 1 << tag;
7005 unsigned long flags;
7006
7007 if (!test_bit(tag, &hba->outstanding_tasks))
7008 goto out;
7009
7010 spin_lock_irqsave(hba->host->host_lock, flags);
7011 ufshcd_utmrl_clear(hba, tag);
7012 spin_unlock_irqrestore(hba->host->host_lock, flags);
7013
7014 /* poll for max. 1 sec to clear door bell register by h/w */
7015 err = ufshcd_wait_for_register(hba,
7016 REG_UTP_TASK_REQ_DOOR_BELL,
7017 mask, 0, 1000, 1000);
7018
7019 dev_err(hba->dev, "Clearing task management function with tag %d %s\n",
7020 tag, err < 0 ? "failed" : "succeeded");
7021
7022 out:
7023 return err;
7024 }
7025
__ufshcd_issue_tm_cmd(struct ufs_hba * hba,struct utp_task_req_desc * treq,u8 tm_function)7026 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
7027 struct utp_task_req_desc *treq, u8 tm_function)
7028 {
7029 struct request_queue *q = hba->tmf_queue;
7030 struct Scsi_Host *host = hba->host;
7031 DECLARE_COMPLETION_ONSTACK(wait);
7032 struct request *req;
7033 unsigned long flags;
7034 int task_tag, err;
7035
7036 /*
7037 * blk_mq_alloc_request() is used here only to get a free tag.
7038 */
7039 req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0);
7040 if (IS_ERR(req))
7041 return PTR_ERR(req);
7042
7043 req->end_io_data = &wait;
7044 ufshcd_hold(hba);
7045
7046 spin_lock_irqsave(host->host_lock, flags);
7047
7048 task_tag = req->tag;
7049 hba->tmf_rqs[req->tag] = req;
7050 treq->upiu_req.req_header.task_tag = task_tag;
7051
7052 memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
7053 ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function);
7054
7055 /* send command to the controller */
7056 __set_bit(task_tag, &hba->outstanding_tasks);
7057 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
7058
7059 spin_unlock_irqrestore(host->host_lock, flags);
7060
7061 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND);
7062
7063 /* wait until the task management command is completed */
7064 err = wait_for_completion_io_timeout(&wait,
7065 msecs_to_jiffies(TM_CMD_TIMEOUT));
7066 if (!err) {
7067 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
7068 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
7069 __func__, tm_function);
7070 if (ufshcd_clear_tm_cmd(hba, task_tag))
7071 dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
7072 __func__, task_tag);
7073 err = -ETIMEDOUT;
7074 } else {
7075 err = 0;
7076 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
7077
7078 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
7079 }
7080
7081 spin_lock_irqsave(hba->host->host_lock, flags);
7082 hba->tmf_rqs[req->tag] = NULL;
7083 __clear_bit(task_tag, &hba->outstanding_tasks);
7084 spin_unlock_irqrestore(hba->host->host_lock, flags);
7085
7086 ufshcd_release(hba);
7087 blk_mq_free_request(req);
7088
7089 return err;
7090 }
7091
7092 /**
7093 * ufshcd_issue_tm_cmd - issues task management commands to controller
7094 * @hba: per adapter instance
7095 * @lun_id: LUN ID to which TM command is sent
7096 * @task_id: task ID to which the TM command is applicable
7097 * @tm_function: task management function opcode
7098 * @tm_response: task management service response return value
7099 *
7100 * Return: non-zero value on error, zero on success.
7101 */
ufshcd_issue_tm_cmd(struct ufs_hba * hba,int lun_id,int task_id,u8 tm_function,u8 * tm_response)7102 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
7103 u8 tm_function, u8 *tm_response)
7104 {
7105 struct utp_task_req_desc treq = { };
7106 enum utp_ocs ocs_value;
7107 int err;
7108
7109 /* Configure task request descriptor */
7110 treq.header.interrupt = 1;
7111 treq.header.ocs = OCS_INVALID_COMMAND_STATUS;
7112
7113 /* Configure task request UPIU */
7114 treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ;
7115 treq.upiu_req.req_header.lun = lun_id;
7116 treq.upiu_req.req_header.tm_function = tm_function;
7117
7118 /*
7119 * The host shall provide the same value for LUN field in the basic
7120 * header and for Input Parameter.
7121 */
7122 treq.upiu_req.input_param1 = cpu_to_be32(lun_id);
7123 treq.upiu_req.input_param2 = cpu_to_be32(task_id);
7124
7125 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
7126 if (err == -ETIMEDOUT)
7127 return err;
7128
7129 ocs_value = treq.header.ocs & MASK_OCS;
7130 if (ocs_value != OCS_SUCCESS)
7131 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
7132 __func__, ocs_value);
7133 else if (tm_response)
7134 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) &
7135 MASK_TM_SERVICE_RESP;
7136 return err;
7137 }
7138
7139 /**
7140 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
7141 * @hba: per-adapter instance
7142 * @req_upiu: upiu request
7143 * @rsp_upiu: upiu reply
7144 * @desc_buff: pointer to descriptor buffer, NULL if NA
7145 * @buff_len: descriptor size, 0 if NA
7146 * @cmd_type: specifies the type (NOP, Query...)
7147 * @desc_op: descriptor operation
7148 *
7149 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
7150 * Therefore, it "rides" the device management infrastructure: uses its tag and
7151 * tasks work queues.
7152 *
7153 * Since there is only one available tag for device management commands,
7154 * the caller is expected to hold the hba->dev_cmd.lock mutex.
7155 *
7156 * Return: 0 upon success; < 0 upon failure.
7157 */
ufshcd_issue_devman_upiu_cmd(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,u8 * desc_buff,int * buff_len,enum dev_cmd_type cmd_type,enum query_opcode desc_op)7158 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
7159 struct utp_upiu_req *req_upiu,
7160 struct utp_upiu_req *rsp_upiu,
7161 u8 *desc_buff, int *buff_len,
7162 enum dev_cmd_type cmd_type,
7163 enum query_opcode desc_op)
7164 {
7165 const u32 tag = hba->reserved_slot;
7166 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7167 int err = 0;
7168 u8 upiu_flags;
7169
7170 /* Protects use of hba->reserved_slot. */
7171 lockdep_assert_held(&hba->dev_cmd.lock);
7172
7173 ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
7174
7175 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
7176
7177 /* update the task tag in the request upiu */
7178 req_upiu->header.task_tag = tag;
7179
7180 /* just copy the upiu request as it is */
7181 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7182 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
7183 /* The Data Segment Area is optional depending upon the query
7184 * function value. for WRITE DESCRIPTOR, the data segment
7185 * follows right after the tsf.
7186 */
7187 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
7188 *buff_len = 0;
7189 }
7190
7191 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7192
7193 /*
7194 * ignore the returning value here - ufshcd_check_query_response is
7195 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
7196 * read the response directly ignoring all errors.
7197 */
7198 ufshcd_issue_dev_cmd(hba, lrbp, tag, QUERY_REQ_TIMEOUT);
7199
7200 /* just copy the upiu response as it is */
7201 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7202 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
7203 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
7204 u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
7205 .data_segment_length);
7206
7207 if (*buff_len >= resp_len) {
7208 memcpy(desc_buff, descp, resp_len);
7209 *buff_len = resp_len;
7210 } else {
7211 dev_warn(hba->dev,
7212 "%s: rsp size %d is bigger than buffer size %d",
7213 __func__, resp_len, *buff_len);
7214 *buff_len = 0;
7215 err = -EINVAL;
7216 }
7217 }
7218 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
7219 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
7220
7221 return err;
7222 }
7223
7224 /**
7225 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
7226 * @hba: per-adapter instance
7227 * @req_upiu: upiu request
7228 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
7229 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
7230 * @desc_buff: pointer to descriptor buffer, NULL if NA
7231 * @buff_len: descriptor size, 0 if NA
7232 * @desc_op: descriptor operation
7233 *
7234 * Supports UTP Transfer requests (nop and query), and UTP Task
7235 * Management requests.
7236 * It is up to the caller to fill the upiu conent properly, as it will
7237 * be copied without any further input validations.
7238 *
7239 * Return: 0 upon success; < 0 upon failure.
7240 */
ufshcd_exec_raw_upiu_cmd(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,enum upiu_request_transaction msgcode,u8 * desc_buff,int * buff_len,enum query_opcode desc_op)7241 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
7242 struct utp_upiu_req *req_upiu,
7243 struct utp_upiu_req *rsp_upiu,
7244 enum upiu_request_transaction msgcode,
7245 u8 *desc_buff, int *buff_len,
7246 enum query_opcode desc_op)
7247 {
7248 int err;
7249 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
7250 struct utp_task_req_desc treq = { };
7251 enum utp_ocs ocs_value;
7252 u8 tm_f = req_upiu->header.tm_function;
7253
7254 switch (msgcode) {
7255 case UPIU_TRANSACTION_NOP_OUT:
7256 cmd_type = DEV_CMD_TYPE_NOP;
7257 fallthrough;
7258 case UPIU_TRANSACTION_QUERY_REQ:
7259 ufshcd_dev_man_lock(hba);
7260 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
7261 desc_buff, buff_len,
7262 cmd_type, desc_op);
7263 ufshcd_dev_man_unlock(hba);
7264
7265 break;
7266 case UPIU_TRANSACTION_TASK_REQ:
7267 treq.header.interrupt = 1;
7268 treq.header.ocs = OCS_INVALID_COMMAND_STATUS;
7269
7270 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu));
7271
7272 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
7273 if (err == -ETIMEDOUT)
7274 break;
7275
7276 ocs_value = treq.header.ocs & MASK_OCS;
7277 if (ocs_value != OCS_SUCCESS) {
7278 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
7279 ocs_value);
7280 break;
7281 }
7282
7283 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu));
7284
7285 break;
7286 default:
7287 err = -EINVAL;
7288
7289 break;
7290 }
7291
7292 return err;
7293 }
7294
7295 /**
7296 * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request
7297 * @hba: per adapter instance
7298 * @req_upiu: upiu request
7299 * @rsp_upiu: upiu reply
7300 * @req_ehs: EHS field which contains Advanced RPMB Request Message
7301 * @rsp_ehs: EHS field which returns Advanced RPMB Response Message
7302 * @sg_cnt: The number of sg lists actually used
7303 * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation
7304 * @dir: DMA direction
7305 *
7306 * Return: zero on success, non-zero on failure.
7307 */
ufshcd_advanced_rpmb_req_handler(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,struct ufs_ehs * req_ehs,struct ufs_ehs * rsp_ehs,int sg_cnt,struct scatterlist * sg_list,enum dma_data_direction dir)7308 int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu,
7309 struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs,
7310 struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list,
7311 enum dma_data_direction dir)
7312 {
7313 const u32 tag = hba->reserved_slot;
7314 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7315 int err = 0;
7316 int result;
7317 u8 upiu_flags;
7318 u8 *ehs_data;
7319 u16 ehs_len;
7320 int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0;
7321
7322 /* Protects use of hba->reserved_slot. */
7323 ufshcd_dev_man_lock(hba);
7324
7325 ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag);
7326
7327 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs);
7328
7329 /* update the task tag */
7330 req_upiu->header.task_tag = tag;
7331
7332 /* copy the UPIU(contains CDB) request as it is */
7333 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7334 /* Copy EHS, starting with byte32, immediately after the CDB package */
7335 memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs));
7336
7337 if (dir != DMA_NONE && sg_list)
7338 ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list);
7339
7340 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7341
7342 err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT);
7343
7344 if (!err) {
7345 /* Just copy the upiu response as it is */
7346 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7347 /* Get the response UPIU result */
7348 result = (lrbp->ucd_rsp_ptr->header.response << 8) |
7349 lrbp->ucd_rsp_ptr->header.status;
7350
7351 ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length;
7352 /*
7353 * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data
7354 * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB
7355 * Message is 02h
7356 */
7357 if (ehs_len == 2 && rsp_ehs) {
7358 /*
7359 * ucd_rsp_ptr points to a buffer with a length of 512 bytes
7360 * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32
7361 */
7362 ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE;
7363 memcpy(rsp_ehs, ehs_data, ehs_len * 32);
7364 }
7365 }
7366
7367 ufshcd_dev_man_unlock(hba);
7368
7369 return err ? : result;
7370 }
7371
7372 /**
7373 * ufshcd_eh_device_reset_handler() - Reset a single logical unit.
7374 * @cmd: SCSI command pointer
7375 *
7376 * Return: SUCCESS or FAILED.
7377 */
ufshcd_eh_device_reset_handler(struct scsi_cmnd * cmd)7378 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7379 {
7380 unsigned long flags, pending_reqs = 0, not_cleared = 0;
7381 struct Scsi_Host *host;
7382 struct ufs_hba *hba;
7383 struct ufs_hw_queue *hwq;
7384 struct ufshcd_lrb *lrbp;
7385 u32 pos, not_cleared_mask = 0;
7386 int err;
7387 u8 resp = 0xF, lun;
7388
7389 host = cmd->device->host;
7390 hba = shost_priv(host);
7391
7392 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
7393 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
7394 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7395 if (!err)
7396 err = resp;
7397 goto out;
7398 }
7399
7400 if (hba->mcq_enabled) {
7401 for (pos = 0; pos < hba->nutrs; pos++) {
7402 lrbp = &hba->lrb[pos];
7403 if (ufshcd_cmd_inflight(lrbp->cmd) &&
7404 lrbp->lun == lun) {
7405 ufshcd_clear_cmd(hba, pos);
7406 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
7407 ufshcd_mcq_poll_cqe_lock(hba, hwq);
7408 }
7409 }
7410 err = 0;
7411 goto out;
7412 }
7413
7414 /* clear the commands that were pending for corresponding LUN */
7415 spin_lock_irqsave(&hba->outstanding_lock, flags);
7416 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
7417 if (hba->lrb[pos].lun == lun)
7418 __set_bit(pos, &pending_reqs);
7419 hba->outstanding_reqs &= ~pending_reqs;
7420 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7421
7422 for_each_set_bit(pos, &pending_reqs, hba->nutrs) {
7423 if (ufshcd_clear_cmd(hba, pos) < 0) {
7424 spin_lock_irqsave(&hba->outstanding_lock, flags);
7425 not_cleared = 1U << pos &
7426 ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7427 hba->outstanding_reqs |= not_cleared;
7428 not_cleared_mask |= not_cleared;
7429 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7430
7431 dev_err(hba->dev, "%s: failed to clear request %d\n",
7432 __func__, pos);
7433 }
7434 }
7435 __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask);
7436
7437 out:
7438 hba->req_abort_count = 0;
7439 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
7440 if (!err) {
7441 err = SUCCESS;
7442 } else {
7443 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7444 err = FAILED;
7445 }
7446 return err;
7447 }
7448
ufshcd_set_req_abort_skip(struct ufs_hba * hba,unsigned long bitmap)7449 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
7450 {
7451 struct ufshcd_lrb *lrbp;
7452 int tag;
7453
7454 for_each_set_bit(tag, &bitmap, hba->nutrs) {
7455 lrbp = &hba->lrb[tag];
7456 lrbp->req_abort_skip = true;
7457 }
7458 }
7459
7460 /**
7461 * ufshcd_try_to_abort_task - abort a specific task
7462 * @hba: Pointer to adapter instance
7463 * @tag: Task tag/index to be aborted
7464 *
7465 * Abort the pending command in device by sending UFS_ABORT_TASK task management
7466 * command, and in host controller by clearing the door-bell register. There can
7467 * be race between controller sending the command to the device while abort is
7468 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
7469 * really issued and then try to abort it.
7470 *
7471 * Return: zero on success, non-zero on failure.
7472 */
ufshcd_try_to_abort_task(struct ufs_hba * hba,int tag)7473 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
7474 {
7475 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7476 int err = 0;
7477 int poll_cnt;
7478 u8 resp = 0xF;
7479 u32 reg;
7480
7481 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
7482 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7483 UFS_QUERY_TASK, &resp);
7484 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
7485 /* cmd pending in the device */
7486 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
7487 __func__, tag);
7488 break;
7489 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7490 /*
7491 * cmd not pending in the device, check if it is
7492 * in transition.
7493 */
7494 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
7495 __func__, tag);
7496 if (hba->mcq_enabled) {
7497 /* MCQ mode */
7498 if (ufshcd_cmd_inflight(lrbp->cmd)) {
7499 /* sleep for max. 200us same delay as in SDB mode */
7500 usleep_range(100, 200);
7501 continue;
7502 }
7503 /* command completed already */
7504 dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
7505 __func__, tag);
7506 goto out;
7507 }
7508
7509 /* Single Doorbell Mode */
7510 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7511 if (reg & (1 << tag)) {
7512 /* sleep for max. 200us to stabilize */
7513 usleep_range(100, 200);
7514 continue;
7515 }
7516 /* command completed already */
7517 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
7518 __func__, tag);
7519 goto out;
7520 } else {
7521 dev_err(hba->dev,
7522 "%s: no response from device. tag = %d, err %d\n",
7523 __func__, tag, err);
7524 if (!err)
7525 err = resp; /* service response error */
7526 goto out;
7527 }
7528 }
7529
7530 if (!poll_cnt) {
7531 err = -EBUSY;
7532 goto out;
7533 }
7534
7535 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7536 UFS_ABORT_TASK, &resp);
7537 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7538 if (!err) {
7539 err = resp; /* service response error */
7540 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
7541 __func__, tag, err);
7542 }
7543 goto out;
7544 }
7545
7546 err = ufshcd_clear_cmd(hba, tag);
7547 if (err)
7548 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
7549 __func__, tag, err);
7550
7551 out:
7552 return err;
7553 }
7554
7555 /**
7556 * ufshcd_abort - scsi host template eh_abort_handler callback
7557 * @cmd: SCSI command pointer
7558 *
7559 * Return: SUCCESS or FAILED.
7560 */
ufshcd_abort(struct scsi_cmnd * cmd)7561 static int ufshcd_abort(struct scsi_cmnd *cmd)
7562 {
7563 struct Scsi_Host *host = cmd->device->host;
7564 struct ufs_hba *hba = shost_priv(host);
7565 int tag = scsi_cmd_to_rq(cmd)->tag;
7566 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7567 unsigned long flags;
7568 int err = FAILED;
7569 bool outstanding;
7570 u32 reg;
7571
7572 ufshcd_hold(hba);
7573
7574 if (!hba->mcq_enabled) {
7575 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7576 if (!test_bit(tag, &hba->outstanding_reqs)) {
7577 /* If command is already aborted/completed, return FAILED. */
7578 dev_err(hba->dev,
7579 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
7580 __func__, tag, hba->outstanding_reqs, reg);
7581 goto release;
7582 }
7583 }
7584
7585 /* Print Transfer Request of aborted task */
7586 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
7587
7588 /*
7589 * Print detailed info about aborted request.
7590 * As more than one request might get aborted at the same time,
7591 * print full information only for the first aborted request in order
7592 * to reduce repeated printouts. For other aborted requests only print
7593 * basic details.
7594 */
7595 scsi_print_command(cmd);
7596 if (!hba->req_abort_count) {
7597 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
7598 ufshcd_print_evt_hist(hba);
7599 ufshcd_print_host_state(hba);
7600 ufshcd_print_pwr_info(hba);
7601 ufshcd_print_tr(hba, tag, true);
7602 } else {
7603 ufshcd_print_tr(hba, tag, false);
7604 }
7605 hba->req_abort_count++;
7606
7607 if (!hba->mcq_enabled && !(reg & (1 << tag))) {
7608 /* only execute this code in single doorbell mode */
7609 dev_err(hba->dev,
7610 "%s: cmd was completed, but without a notifying intr, tag = %d",
7611 __func__, tag);
7612 __ufshcd_transfer_req_compl(hba, 1UL << tag);
7613 goto release;
7614 }
7615
7616 /*
7617 * Task abort to the device W-LUN is illegal. When this command
7618 * will fail, due to spec violation, scsi err handling next step
7619 * will be to send LU reset which, again, is a spec violation.
7620 * To avoid these unnecessary/illegal steps, first we clean up
7621 * the lrb taken by this cmd and re-set it in outstanding_reqs,
7622 * then queue the eh_work and bail.
7623 */
7624 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
7625 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
7626
7627 spin_lock_irqsave(host->host_lock, flags);
7628 hba->force_reset = true;
7629 ufshcd_schedule_eh_work(hba);
7630 spin_unlock_irqrestore(host->host_lock, flags);
7631 goto release;
7632 }
7633
7634 if (hba->mcq_enabled) {
7635 /* MCQ mode. Branch off to handle abort for mcq mode */
7636 err = ufshcd_mcq_abort(cmd);
7637 goto release;
7638 }
7639
7640 /* Skip task abort in case previous aborts failed and report failure */
7641 if (lrbp->req_abort_skip) {
7642 dev_err(hba->dev, "%s: skipping abort\n", __func__);
7643 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7644 goto release;
7645 }
7646
7647 err = ufshcd_try_to_abort_task(hba, tag);
7648 if (err) {
7649 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7650 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7651 err = FAILED;
7652 goto release;
7653 }
7654
7655 /*
7656 * Clear the corresponding bit from outstanding_reqs since the command
7657 * has been aborted successfully.
7658 */
7659 spin_lock_irqsave(&hba->outstanding_lock, flags);
7660 outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);
7661 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7662
7663 if (outstanding)
7664 ufshcd_release_scsi_cmd(hba, lrbp);
7665
7666 err = SUCCESS;
7667
7668 release:
7669 /* Matches the ufshcd_hold() call at the start of this function. */
7670 ufshcd_release(hba);
7671 return err;
7672 }
7673
7674 /**
7675 * ufshcd_host_reset_and_restore - reset and restore host controller
7676 * @hba: per-adapter instance
7677 *
7678 * Note that host controller reset may issue DME_RESET to
7679 * local and remote (device) Uni-Pro stack and the attributes
7680 * are reset to default state.
7681 *
7682 * Return: zero on success, non-zero on failure.
7683 */
ufshcd_host_reset_and_restore(struct ufs_hba * hba)7684 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
7685 {
7686 int err;
7687
7688 /*
7689 * Stop the host controller and complete the requests
7690 * cleared by h/w
7691 */
7692 ufshcd_hba_stop(hba);
7693 hba->silence_err_logs = true;
7694 ufshcd_complete_requests(hba, true);
7695 hba->silence_err_logs = false;
7696
7697 /* scale up clocks to max frequency before full reinitialization */
7698 ufshcd_scale_clks(hba, ULONG_MAX, true);
7699
7700 err = ufshcd_hba_enable(hba);
7701
7702 /* Establish the link again and restore the device */
7703 if (!err)
7704 err = ufshcd_probe_hba(hba, false);
7705
7706 if (err)
7707 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
7708 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
7709 return err;
7710 }
7711
7712 /**
7713 * ufshcd_reset_and_restore - reset and re-initialize host/device
7714 * @hba: per-adapter instance
7715 *
7716 * Reset and recover device, host and re-establish link. This
7717 * is helpful to recover the communication in fatal error conditions.
7718 *
7719 * Return: zero on success, non-zero on failure.
7720 */
ufshcd_reset_and_restore(struct ufs_hba * hba)7721 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7722 {
7723 u32 saved_err = 0;
7724 u32 saved_uic_err = 0;
7725 int err = 0;
7726 unsigned long flags;
7727 int retries = MAX_HOST_RESET_RETRIES;
7728
7729 spin_lock_irqsave(hba->host->host_lock, flags);
7730 do {
7731 /*
7732 * This is a fresh start, cache and clear saved error first,
7733 * in case new error generated during reset and restore.
7734 */
7735 saved_err |= hba->saved_err;
7736 saved_uic_err |= hba->saved_uic_err;
7737 hba->saved_err = 0;
7738 hba->saved_uic_err = 0;
7739 hba->force_reset = false;
7740 hba->ufshcd_state = UFSHCD_STATE_RESET;
7741 spin_unlock_irqrestore(hba->host->host_lock, flags);
7742
7743 /* Reset the attached device */
7744 ufshcd_device_reset(hba);
7745
7746 err = ufshcd_host_reset_and_restore(hba);
7747
7748 spin_lock_irqsave(hba->host->host_lock, flags);
7749 if (err)
7750 continue;
7751 /* Do not exit unless operational or dead */
7752 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
7753 hba->ufshcd_state != UFSHCD_STATE_ERROR &&
7754 hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL)
7755 err = -EAGAIN;
7756 } while (err && --retries);
7757
7758 /*
7759 * Inform scsi mid-layer that we did reset and allow to handle
7760 * Unit Attention properly.
7761 */
7762 scsi_report_bus_reset(hba->host, 0);
7763 if (err) {
7764 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7765 hba->saved_err |= saved_err;
7766 hba->saved_uic_err |= saved_uic_err;
7767 }
7768 spin_unlock_irqrestore(hba->host->host_lock, flags);
7769
7770 return err;
7771 }
7772
7773 /**
7774 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
7775 * @cmd: SCSI command pointer
7776 *
7777 * Return: SUCCESS or FAILED.
7778 */
ufshcd_eh_host_reset_handler(struct scsi_cmnd * cmd)7779 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7780 {
7781 int err = SUCCESS;
7782 unsigned long flags;
7783 struct ufs_hba *hba;
7784
7785 hba = shost_priv(cmd->device->host);
7786
7787 /*
7788 * If runtime PM sent SSU and got a timeout, scsi_error_handler is
7789 * stuck in this function waiting for flush_work(&hba->eh_work). And
7790 * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do
7791 * ufshcd_link_recovery instead of eh_work to prevent deadlock.
7792 */
7793 if (hba->pm_op_in_progress) {
7794 if (ufshcd_link_recovery(hba))
7795 err = FAILED;
7796
7797 return err;
7798 }
7799
7800 spin_lock_irqsave(hba->host->host_lock, flags);
7801 hba->force_reset = true;
7802 ufshcd_schedule_eh_work(hba);
7803 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
7804 spin_unlock_irqrestore(hba->host->host_lock, flags);
7805
7806 flush_work(&hba->eh_work);
7807
7808 spin_lock_irqsave(hba->host->host_lock, flags);
7809 if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
7810 err = FAILED;
7811 spin_unlock_irqrestore(hba->host->host_lock, flags);
7812
7813 return err;
7814 }
7815
7816 /**
7817 * ufshcd_get_max_icc_level - calculate the ICC level
7818 * @sup_curr_uA: max. current supported by the regulator
7819 * @start_scan: row at the desc table to start scan from
7820 * @buff: power descriptor buffer
7821 *
7822 * Return: calculated max ICC level for specific regulator.
7823 */
ufshcd_get_max_icc_level(int sup_curr_uA,u32 start_scan,const char * buff)7824 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan,
7825 const char *buff)
7826 {
7827 int i;
7828 int curr_uA;
7829 u16 data;
7830 u16 unit;
7831
7832 for (i = start_scan; i >= 0; i--) {
7833 data = get_unaligned_be16(&buff[2 * i]);
7834 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7835 ATTR_ICC_LVL_UNIT_OFFSET;
7836 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7837 switch (unit) {
7838 case UFSHCD_NANO_AMP:
7839 curr_uA = curr_uA / 1000;
7840 break;
7841 case UFSHCD_MILI_AMP:
7842 curr_uA = curr_uA * 1000;
7843 break;
7844 case UFSHCD_AMP:
7845 curr_uA = curr_uA * 1000 * 1000;
7846 break;
7847 case UFSHCD_MICRO_AMP:
7848 default:
7849 break;
7850 }
7851 if (sup_curr_uA >= curr_uA)
7852 break;
7853 }
7854 if (i < 0) {
7855 i = 0;
7856 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7857 }
7858
7859 return (u32)i;
7860 }
7861
7862 /**
7863 * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level
7864 * In case regulators are not initialized we'll return 0
7865 * @hba: per-adapter instance
7866 * @desc_buf: power descriptor buffer to extract ICC levels from.
7867 *
7868 * Return: calculated ICC level.
7869 */
ufshcd_find_max_sup_active_icc_level(struct ufs_hba * hba,const u8 * desc_buf)7870 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7871 const u8 *desc_buf)
7872 {
7873 u32 icc_level = 0;
7874
7875 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7876 !hba->vreg_info.vccq2) {
7877 /*
7878 * Using dev_dbg to avoid messages during runtime PM to avoid
7879 * never-ending cycles of messages written back to storage by
7880 * user space causing runtime resume, causing more messages and
7881 * so on.
7882 */
7883 dev_dbg(hba->dev,
7884 "%s: Regulator capability was not set, actvIccLevel=%d",
7885 __func__, icc_level);
7886 goto out;
7887 }
7888
7889 if (hba->vreg_info.vcc->max_uA)
7890 icc_level = ufshcd_get_max_icc_level(
7891 hba->vreg_info.vcc->max_uA,
7892 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7893 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7894
7895 if (hba->vreg_info.vccq->max_uA)
7896 icc_level = ufshcd_get_max_icc_level(
7897 hba->vreg_info.vccq->max_uA,
7898 icc_level,
7899 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7900
7901 if (hba->vreg_info.vccq2->max_uA)
7902 icc_level = ufshcd_get_max_icc_level(
7903 hba->vreg_info.vccq2->max_uA,
7904 icc_level,
7905 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7906 out:
7907 return icc_level;
7908 }
7909
ufshcd_set_active_icc_lvl(struct ufs_hba * hba)7910 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
7911 {
7912 int ret;
7913 u8 *desc_buf;
7914 u32 icc_level;
7915
7916 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
7917 if (!desc_buf)
7918 return;
7919
7920 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7921 desc_buf, QUERY_DESC_MAX_SIZE);
7922 if (ret) {
7923 dev_err(hba->dev,
7924 "%s: Failed reading power descriptor ret = %d",
7925 __func__, ret);
7926 goto out;
7927 }
7928
7929 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf);
7930 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7931
7932 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7933 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
7934
7935 if (ret)
7936 dev_err(hba->dev,
7937 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
7938 __func__, icc_level, ret);
7939
7940 out:
7941 kfree(desc_buf);
7942 }
7943
ufshcd_blk_pm_runtime_init(struct scsi_device * sdev)7944 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7945 {
7946 struct Scsi_Host *shost = sdev->host;
7947
7948 scsi_autopm_get_device(sdev);
7949 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7950 if (sdev->rpm_autosuspend)
7951 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7952 shost->rpm_autosuspend_delay);
7953 scsi_autopm_put_device(sdev);
7954 }
7955
7956 /**
7957 * ufshcd_scsi_add_wlus - Adds required W-LUs
7958 * @hba: per-adapter instance
7959 *
7960 * UFS device specification requires the UFS devices to support 4 well known
7961 * logical units:
7962 * "REPORT_LUNS" (address: 01h)
7963 * "UFS Device" (address: 50h)
7964 * "RPMB" (address: 44h)
7965 * "BOOT" (address: 30h)
7966 * UFS device's power management needs to be controlled by "POWER CONDITION"
7967 * field of SSU (START STOP UNIT) command. But this "power condition" field
7968 * will take effect only when its sent to "UFS device" well known logical unit
7969 * hence we require the scsi_device instance to represent this logical unit in
7970 * order for the UFS host driver to send the SSU command for power management.
7971 *
7972 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7973 * Block) LU so user space process can control this LU. User space may also
7974 * want to have access to BOOT LU.
7975 *
7976 * This function adds scsi device instances for each of all well known LUs
7977 * (except "REPORT LUNS" LU).
7978 *
7979 * Return: zero on success (all required W-LUs are added successfully),
7980 * non-zero error value on failure (if failed to add any of the required W-LU).
7981 */
ufshcd_scsi_add_wlus(struct ufs_hba * hba)7982 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7983 {
7984 int ret = 0;
7985 struct scsi_device *sdev_boot, *sdev_rpmb;
7986
7987 hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0,
7988 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7989 if (IS_ERR(hba->ufs_device_wlun)) {
7990 ret = PTR_ERR(hba->ufs_device_wlun);
7991 hba->ufs_device_wlun = NULL;
7992 goto out;
7993 }
7994 scsi_device_put(hba->ufs_device_wlun);
7995
7996 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
7997 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7998 if (IS_ERR(sdev_rpmb)) {
7999 ret = PTR_ERR(sdev_rpmb);
8000 goto remove_ufs_device_wlun;
8001 }
8002 ufshcd_blk_pm_runtime_init(sdev_rpmb);
8003 scsi_device_put(sdev_rpmb);
8004
8005 sdev_boot = __scsi_add_device(hba->host, 0, 0,
8006 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
8007 if (IS_ERR(sdev_boot)) {
8008 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
8009 } else {
8010 ufshcd_blk_pm_runtime_init(sdev_boot);
8011 scsi_device_put(sdev_boot);
8012 }
8013 goto out;
8014
8015 remove_ufs_device_wlun:
8016 scsi_remove_device(hba->ufs_device_wlun);
8017 out:
8018 return ret;
8019 }
8020
ufshcd_wb_probe(struct ufs_hba * hba,const u8 * desc_buf)8021 static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf)
8022 {
8023 struct ufs_dev_info *dev_info = &hba->dev_info;
8024 u8 lun;
8025 u32 d_lu_wb_buf_alloc;
8026 u32 ext_ufs_feature;
8027
8028 if (!ufshcd_is_wb_allowed(hba))
8029 return;
8030
8031 /*
8032 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
8033 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
8034 * enabled
8035 */
8036 if (!(dev_info->wspecversion >= 0x310 ||
8037 dev_info->wspecversion == 0x220 ||
8038 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
8039 goto wb_disabled;
8040
8041 ext_ufs_feature = get_unaligned_be32(desc_buf +
8042 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8043
8044 if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP))
8045 goto wb_disabled;
8046
8047 /*
8048 * WB may be supported but not configured while provisioning. The spec
8049 * says, in dedicated wb buffer mode, a max of 1 lun would have wb
8050 * buffer configured.
8051 */
8052 dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
8053
8054 dev_info->b_presrv_uspc_en =
8055 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
8056
8057 if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
8058 if (!get_unaligned_be32(desc_buf +
8059 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS))
8060 goto wb_disabled;
8061 } else {
8062 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
8063 d_lu_wb_buf_alloc = 0;
8064 ufshcd_read_unit_desc_param(hba,
8065 lun,
8066 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
8067 (u8 *)&d_lu_wb_buf_alloc,
8068 sizeof(d_lu_wb_buf_alloc));
8069 if (d_lu_wb_buf_alloc) {
8070 dev_info->wb_dedicated_lu = lun;
8071 break;
8072 }
8073 }
8074
8075 if (!d_lu_wb_buf_alloc)
8076 goto wb_disabled;
8077 }
8078
8079 if (!ufshcd_is_wb_buf_lifetime_available(hba))
8080 goto wb_disabled;
8081
8082 return;
8083
8084 wb_disabled:
8085 hba->caps &= ~UFSHCD_CAP_WB_EN;
8086 }
8087
ufshcd_temp_notif_probe(struct ufs_hba * hba,const u8 * desc_buf)8088 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf)
8089 {
8090 struct ufs_dev_info *dev_info = &hba->dev_info;
8091 u32 ext_ufs_feature;
8092 u8 mask = 0;
8093
8094 if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300)
8095 return;
8096
8097 ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8098
8099 if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF)
8100 mask |= MASK_EE_TOO_LOW_TEMP;
8101
8102 if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF)
8103 mask |= MASK_EE_TOO_HIGH_TEMP;
8104
8105 if (mask) {
8106 ufshcd_enable_ee(hba, mask);
8107 ufs_hwmon_probe(hba, mask);
8108 }
8109 }
8110
ufshcd_ext_iid_probe(struct ufs_hba * hba,u8 * desc_buf)8111 static void ufshcd_ext_iid_probe(struct ufs_hba *hba, u8 *desc_buf)
8112 {
8113 struct ufs_dev_info *dev_info = &hba->dev_info;
8114 u32 ext_ufs_feature;
8115 u32 ext_iid_en = 0;
8116 int err;
8117
8118 /* Only UFS-4.0 and above may support EXT_IID */
8119 if (dev_info->wspecversion < 0x400)
8120 goto out;
8121
8122 ext_ufs_feature = get_unaligned_be32(desc_buf +
8123 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8124 if (!(ext_ufs_feature & UFS_DEV_EXT_IID_SUP))
8125 goto out;
8126
8127 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8128 QUERY_ATTR_IDN_EXT_IID_EN, 0, 0, &ext_iid_en);
8129 if (err)
8130 dev_err(hba->dev, "failed reading bEXTIIDEn. err = %d\n", err);
8131
8132 out:
8133 dev_info->b_ext_iid_en = ext_iid_en;
8134 }
8135
ufshcd_set_rtt(struct ufs_hba * hba)8136 static void ufshcd_set_rtt(struct ufs_hba *hba)
8137 {
8138 struct ufs_dev_info *dev_info = &hba->dev_info;
8139 u32 rtt = 0;
8140 u32 dev_rtt = 0;
8141 int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ?
8142 hba->vops->max_num_rtt : hba->nortt;
8143
8144 /* RTT override makes sense only for UFS-4.0 and above */
8145 if (dev_info->wspecversion < 0x400)
8146 return;
8147
8148 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8149 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) {
8150 dev_err(hba->dev, "failed reading bMaxNumOfRTT\n");
8151 return;
8152 }
8153
8154 /* do not override if it was already written */
8155 if (dev_rtt != DEFAULT_MAX_NUM_RTT)
8156 return;
8157
8158 rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap);
8159
8160 if (rtt == dev_rtt)
8161 return;
8162
8163 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8164 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt))
8165 dev_err(hba->dev, "failed writing bMaxNumOfRTT\n");
8166 }
8167
ufshcd_fixup_dev_quirks(struct ufs_hba * hba,const struct ufs_dev_quirk * fixups)8168 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba,
8169 const struct ufs_dev_quirk *fixups)
8170 {
8171 const struct ufs_dev_quirk *f;
8172 struct ufs_dev_info *dev_info = &hba->dev_info;
8173
8174 if (!fixups)
8175 return;
8176
8177 for (f = fixups; f->quirk; f++) {
8178 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
8179 f->wmanufacturerid == UFS_ANY_VENDOR) &&
8180 ((dev_info->model &&
8181 STR_PRFX_EQUAL(f->model, dev_info->model)) ||
8182 !strcmp(f->model, UFS_ANY_MODEL)))
8183 hba->dev_quirks |= f->quirk;
8184 }
8185 }
8186 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
8187
ufs_fixup_device_setup(struct ufs_hba * hba)8188 static void ufs_fixup_device_setup(struct ufs_hba *hba)
8189 {
8190 /* fix by general quirk table */
8191 ufshcd_fixup_dev_quirks(hba, ufs_fixups);
8192
8193 /* allow vendors to fix quirks */
8194 ufshcd_vops_fixup_dev_quirks(hba);
8195 }
8196
ufshcd_update_rtc(struct ufs_hba * hba)8197 static void ufshcd_update_rtc(struct ufs_hba *hba)
8198 {
8199 struct timespec64 ts64;
8200 int err;
8201 u32 val;
8202
8203 ktime_get_real_ts64(&ts64);
8204
8205 if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) {
8206 dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__);
8207 return;
8208 }
8209
8210 /*
8211 * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond
8212 * 2146 is required, it is recommended to choose the relative RTC mode.
8213 */
8214 val = ts64.tv_sec - hba->dev_info.rtc_time_baseline;
8215
8216 /* Skip update RTC if RPM state is not RPM_ACTIVE */
8217 if (ufshcd_rpm_get_if_active(hba) <= 0)
8218 return;
8219
8220 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED,
8221 0, 0, &val);
8222 ufshcd_rpm_put(hba);
8223
8224 if (err)
8225 dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err);
8226 else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE)
8227 hba->dev_info.rtc_time_baseline = ts64.tv_sec;
8228 }
8229
ufshcd_rtc_work(struct work_struct * work)8230 static void ufshcd_rtc_work(struct work_struct *work)
8231 {
8232 struct ufs_hba *hba;
8233
8234 hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work);
8235
8236 /* Update RTC only when there are no requests in progress and UFSHCI is operational */
8237 if (!ufshcd_is_ufs_dev_busy(hba) && hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL)
8238 ufshcd_update_rtc(hba);
8239
8240 if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period)
8241 schedule_delayed_work(&hba->ufs_rtc_update_work,
8242 msecs_to_jiffies(hba->dev_info.rtc_update_period));
8243 }
8244
ufs_init_rtc(struct ufs_hba * hba,u8 * desc_buf)8245 static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf)
8246 {
8247 u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]);
8248 struct ufs_dev_info *dev_info = &hba->dev_info;
8249
8250 if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) {
8251 dev_info->rtc_type = UFS_RTC_ABSOLUTE;
8252
8253 /*
8254 * The concept of measuring time in Linux as the number of seconds elapsed since
8255 * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st
8256 * 2010 00:00, here we need to adjust ABS baseline.
8257 */
8258 dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) -
8259 mktime64(1970, 1, 1, 0, 0, 0);
8260 } else {
8261 dev_info->rtc_type = UFS_RTC_RELATIVE;
8262 dev_info->rtc_time_baseline = 0;
8263 }
8264
8265 /*
8266 * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state
8267 * how to calculate the specific update period for each time unit. And we disable periodic
8268 * RTC update work, let user configure by sysfs node according to specific circumstance.
8269 */
8270 dev_info->rtc_update_period = 0;
8271 }
8272
ufs_get_device_desc(struct ufs_hba * hba)8273 static int ufs_get_device_desc(struct ufs_hba *hba)
8274 {
8275 int err;
8276 u8 model_index;
8277 u8 *desc_buf;
8278 struct ufs_dev_info *dev_info = &hba->dev_info;
8279
8280 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
8281 if (!desc_buf) {
8282 err = -ENOMEM;
8283 goto out;
8284 }
8285
8286 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
8287 QUERY_DESC_MAX_SIZE);
8288 if (err) {
8289 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
8290 __func__, err);
8291 goto out;
8292 }
8293
8294 /*
8295 * getting vendor (manufacturerID) and Bank Index in big endian
8296 * format
8297 */
8298 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
8299 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
8300
8301 /* getting Specification Version in big endian format */
8302 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
8303 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
8304 dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH];
8305
8306 dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP];
8307
8308 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
8309
8310 err = ufshcd_read_string_desc(hba, model_index,
8311 &dev_info->model, SD_ASCII_STD);
8312 if (err < 0) {
8313 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
8314 __func__, err);
8315 goto out;
8316 }
8317
8318 hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
8319 desc_buf[DEVICE_DESC_PARAM_NUM_WLU];
8320
8321 ufs_fixup_device_setup(hba);
8322
8323 ufshcd_wb_probe(hba, desc_buf);
8324
8325 ufshcd_temp_notif_probe(hba, desc_buf);
8326
8327 ufs_init_rtc(hba, desc_buf);
8328
8329 if (hba->ext_iid_sup)
8330 ufshcd_ext_iid_probe(hba, desc_buf);
8331
8332 /*
8333 * ufshcd_read_string_desc returns size of the string
8334 * reset the error value
8335 */
8336 err = 0;
8337
8338 out:
8339 kfree(desc_buf);
8340 return err;
8341 }
8342
ufs_put_device_desc(struct ufs_hba * hba)8343 static void ufs_put_device_desc(struct ufs_hba *hba)
8344 {
8345 struct ufs_dev_info *dev_info = &hba->dev_info;
8346
8347 kfree(dev_info->model);
8348 dev_info->model = NULL;
8349 }
8350
8351 /**
8352 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
8353 * less than device PA_TACTIVATE time.
8354 * @hba: per-adapter instance
8355 *
8356 * Some UFS devices require host PA_TACTIVATE to be lower than device
8357 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
8358 * for such devices.
8359 *
8360 * Return: zero on success, non-zero error value on failure.
8361 */
ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba * hba)8362 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
8363 {
8364 int ret = 0;
8365 u32 granularity, peer_granularity;
8366 u32 pa_tactivate, peer_pa_tactivate;
8367 u32 pa_tactivate_us, peer_pa_tactivate_us;
8368 static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
8369
8370 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
8371 &granularity);
8372 if (ret)
8373 goto out;
8374
8375 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
8376 &peer_granularity);
8377 if (ret)
8378 goto out;
8379
8380 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
8381 (granularity > PA_GRANULARITY_MAX_VAL)) {
8382 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
8383 __func__, granularity);
8384 return -EINVAL;
8385 }
8386
8387 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
8388 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
8389 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
8390 __func__, peer_granularity);
8391 return -EINVAL;
8392 }
8393
8394 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
8395 if (ret)
8396 goto out;
8397
8398 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
8399 &peer_pa_tactivate);
8400 if (ret)
8401 goto out;
8402
8403 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
8404 peer_pa_tactivate_us = peer_pa_tactivate *
8405 gran_to_us_table[peer_granularity - 1];
8406
8407 if (pa_tactivate_us >= peer_pa_tactivate_us) {
8408 u32 new_peer_pa_tactivate;
8409
8410 new_peer_pa_tactivate = pa_tactivate_us /
8411 gran_to_us_table[peer_granularity - 1];
8412 new_peer_pa_tactivate++;
8413 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
8414 new_peer_pa_tactivate);
8415 }
8416
8417 out:
8418 return ret;
8419 }
8420
ufshcd_tune_unipro_params(struct ufs_hba * hba)8421 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
8422 {
8423 ufshcd_vops_apply_dev_quirks(hba);
8424
8425 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
8426 /* set 1ms timeout for PA_TACTIVATE */
8427 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
8428
8429 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
8430 ufshcd_quirk_tune_host_pa_tactivate(hba);
8431 }
8432
ufshcd_clear_dbg_ufs_stats(struct ufs_hba * hba)8433 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
8434 {
8435 hba->ufs_stats.hibern8_exit_cnt = 0;
8436 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
8437 hba->req_abort_count = 0;
8438 }
8439
ufshcd_device_geo_params_init(struct ufs_hba * hba)8440 static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
8441 {
8442 int err;
8443 u8 *desc_buf;
8444
8445 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
8446 if (!desc_buf) {
8447 err = -ENOMEM;
8448 goto out;
8449 }
8450
8451 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
8452 desc_buf, QUERY_DESC_MAX_SIZE);
8453 if (err) {
8454 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
8455 __func__, err);
8456 goto out;
8457 }
8458
8459 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
8460 hba->dev_info.max_lu_supported = 32;
8461 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
8462 hba->dev_info.max_lu_supported = 8;
8463
8464 out:
8465 kfree(desc_buf);
8466 return err;
8467 }
8468
8469 struct ufs_ref_clk {
8470 unsigned long freq_hz;
8471 enum ufs_ref_clk_freq val;
8472 };
8473
8474 static const struct ufs_ref_clk ufs_ref_clk_freqs[] = {
8475 {19200000, REF_CLK_FREQ_19_2_MHZ},
8476 {26000000, REF_CLK_FREQ_26_MHZ},
8477 {38400000, REF_CLK_FREQ_38_4_MHZ},
8478 {52000000, REF_CLK_FREQ_52_MHZ},
8479 {0, REF_CLK_FREQ_INVAL},
8480 };
8481
8482 static enum ufs_ref_clk_freq
ufs_get_bref_clk_from_hz(unsigned long freq)8483 ufs_get_bref_clk_from_hz(unsigned long freq)
8484 {
8485 int i;
8486
8487 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
8488 if (ufs_ref_clk_freqs[i].freq_hz == freq)
8489 return ufs_ref_clk_freqs[i].val;
8490
8491 return REF_CLK_FREQ_INVAL;
8492 }
8493
ufshcd_parse_dev_ref_clk_freq(struct ufs_hba * hba,struct clk * refclk)8494 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
8495 {
8496 unsigned long freq;
8497
8498 freq = clk_get_rate(refclk);
8499
8500 hba->dev_ref_clk_freq =
8501 ufs_get_bref_clk_from_hz(freq);
8502
8503 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
8504 dev_err(hba->dev,
8505 "invalid ref_clk setting = %ld\n", freq);
8506 }
8507
ufshcd_set_dev_ref_clk(struct ufs_hba * hba)8508 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
8509 {
8510 int err;
8511 u32 ref_clk;
8512 u32 freq = hba->dev_ref_clk_freq;
8513
8514 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8515 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
8516
8517 if (err) {
8518 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
8519 err);
8520 goto out;
8521 }
8522
8523 if (ref_clk == freq)
8524 goto out; /* nothing to update */
8525
8526 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8527 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
8528
8529 if (err) {
8530 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
8531 ufs_ref_clk_freqs[freq].freq_hz);
8532 goto out;
8533 }
8534
8535 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
8536 ufs_ref_clk_freqs[freq].freq_hz);
8537
8538 out:
8539 return err;
8540 }
8541
ufshcd_device_params_init(struct ufs_hba * hba)8542 static int ufshcd_device_params_init(struct ufs_hba *hba)
8543 {
8544 bool flag;
8545 int ret;
8546
8547 /* Init UFS geometry descriptor related parameters */
8548 ret = ufshcd_device_geo_params_init(hba);
8549 if (ret)
8550 goto out;
8551
8552 /* Check and apply UFS device quirks */
8553 ret = ufs_get_device_desc(hba);
8554 if (ret) {
8555 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
8556 __func__, ret);
8557 goto out;
8558 }
8559
8560 ufshcd_set_rtt(hba);
8561
8562 ufshcd_get_ref_clk_gating_wait(hba);
8563
8564 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
8565 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
8566 hba->dev_info.f_power_on_wp_en = flag;
8567
8568 /* Probe maximum power mode co-supported by both UFS host and device */
8569 if (ufshcd_get_max_pwr_mode(hba))
8570 dev_err(hba->dev,
8571 "%s: Failed getting max supported power mode\n",
8572 __func__);
8573 out:
8574 return ret;
8575 }
8576
ufshcd_set_timestamp_attr(struct ufs_hba * hba)8577 static void ufshcd_set_timestamp_attr(struct ufs_hba *hba)
8578 {
8579 int err;
8580 struct ufs_query_req *request = NULL;
8581 struct ufs_query_res *response = NULL;
8582 struct ufs_dev_info *dev_info = &hba->dev_info;
8583 struct utp_upiu_query_v4_0 *upiu_data;
8584
8585 if (dev_info->wspecversion < 0x400)
8586 return;
8587
8588 ufshcd_dev_man_lock(hba);
8589
8590 ufshcd_init_query(hba, &request, &response,
8591 UPIU_QUERY_OPCODE_WRITE_ATTR,
8592 QUERY_ATTR_IDN_TIMESTAMP, 0, 0);
8593
8594 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
8595
8596 upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req;
8597
8598 put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3);
8599
8600 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
8601
8602 if (err)
8603 dev_err(hba->dev, "%s: failed to set timestamp %d\n",
8604 __func__, err);
8605
8606 ufshcd_dev_man_unlock(hba);
8607 }
8608
8609 /**
8610 * ufshcd_add_lus - probe and add UFS logical units
8611 * @hba: per-adapter instance
8612 *
8613 * Return: 0 upon success; < 0 upon failure.
8614 */
ufshcd_add_lus(struct ufs_hba * hba)8615 static int ufshcd_add_lus(struct ufs_hba *hba)
8616 {
8617 int ret;
8618
8619 /* Add required well known logical units to scsi mid layer */
8620 ret = ufshcd_scsi_add_wlus(hba);
8621 if (ret)
8622 goto out;
8623
8624 /* Initialize devfreq after UFS device is detected */
8625 if (ufshcd_is_clkscaling_supported(hba)) {
8626 memcpy(&hba->clk_scaling.saved_pwr_info,
8627 &hba->pwr_info,
8628 sizeof(struct ufs_pa_layer_attr));
8629 hba->clk_scaling.is_allowed = true;
8630
8631 ret = ufshcd_devfreq_init(hba);
8632 if (ret)
8633 goto out;
8634
8635 hba->clk_scaling.is_enabled = true;
8636 ufshcd_init_clk_scaling_sysfs(hba);
8637 }
8638
8639 ufs_bsg_probe(hba);
8640 scsi_scan_host(hba->host);
8641
8642 out:
8643 return ret;
8644 }
8645
8646 /* SDB - Single Doorbell */
ufshcd_release_sdb_queue(struct ufs_hba * hba,int nutrs)8647 static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs)
8648 {
8649 size_t ucdl_size, utrdl_size;
8650
8651 ucdl_size = ufshcd_get_ucd_size(hba) * nutrs;
8652 dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr,
8653 hba->ucdl_dma_addr);
8654
8655 utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs;
8656 dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr,
8657 hba->utrdl_dma_addr);
8658
8659 devm_kfree(hba->dev, hba->lrb);
8660 }
8661
ufshcd_alloc_mcq(struct ufs_hba * hba)8662 static int ufshcd_alloc_mcq(struct ufs_hba *hba)
8663 {
8664 int ret;
8665 int old_nutrs = hba->nutrs;
8666
8667 ret = ufshcd_mcq_decide_queue_depth(hba);
8668 if (ret < 0)
8669 return ret;
8670
8671 hba->nutrs = ret;
8672 ret = ufshcd_mcq_init(hba);
8673 if (ret)
8674 goto err;
8675
8676 /*
8677 * Previously allocated memory for nutrs may not be enough in MCQ mode.
8678 * Number of supported tags in MCQ mode may be larger than SDB mode.
8679 */
8680 if (hba->nutrs != old_nutrs) {
8681 ufshcd_release_sdb_queue(hba, old_nutrs);
8682 ret = ufshcd_memory_alloc(hba);
8683 if (ret)
8684 goto err;
8685 ufshcd_host_memory_configure(hba);
8686 }
8687
8688 ret = ufshcd_mcq_memory_alloc(hba);
8689 if (ret)
8690 goto err;
8691
8692 hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
8693 hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED;
8694
8695 return 0;
8696 err:
8697 hba->nutrs = old_nutrs;
8698 return ret;
8699 }
8700
ufshcd_config_mcq(struct ufs_hba * hba)8701 static void ufshcd_config_mcq(struct ufs_hba *hba)
8702 {
8703 int ret;
8704 u32 intrs;
8705
8706 ret = ufshcd_mcq_vops_config_esi(hba);
8707 dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : "");
8708
8709 intrs = UFSHCD_ENABLE_MCQ_INTRS;
8710 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR)
8711 intrs &= ~MCQ_CQ_EVENT_STATUS;
8712 ufshcd_enable_intr(hba, intrs);
8713 ufshcd_mcq_make_queues_operational(hba);
8714 ufshcd_mcq_config_mac(hba, hba->nutrs);
8715
8716 dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n",
8717 hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT],
8718 hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL],
8719 hba->nutrs);
8720 }
8721
ufshcd_device_init(struct ufs_hba * hba,bool init_dev_params)8722 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params)
8723 {
8724 int ret;
8725 struct Scsi_Host *host = hba->host;
8726
8727 hba->ufshcd_state = UFSHCD_STATE_RESET;
8728
8729 ret = ufshcd_link_startup(hba);
8730 if (ret)
8731 return ret;
8732
8733 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION)
8734 return ret;
8735
8736 /* Debug counters initialization */
8737 ufshcd_clear_dbg_ufs_stats(hba);
8738
8739 /* UniPro link is active now */
8740 ufshcd_set_link_active(hba);
8741
8742 /* Reconfigure MCQ upon reset */
8743 if (hba->mcq_enabled && !init_dev_params) {
8744 ufshcd_config_mcq(hba);
8745 ufshcd_mcq_enable(hba);
8746 }
8747
8748 /* Verify device initialization by sending NOP OUT UPIU */
8749 ret = ufshcd_verify_dev_init(hba);
8750 if (ret)
8751 return ret;
8752
8753 /* Initiate UFS initialization, and waiting until completion */
8754 ret = ufshcd_complete_dev_init(hba);
8755 if (ret)
8756 return ret;
8757
8758 /*
8759 * Initialize UFS device parameters used by driver, these
8760 * parameters are associated with UFS descriptors.
8761 */
8762 if (init_dev_params) {
8763 ret = ufshcd_device_params_init(hba);
8764 if (ret)
8765 return ret;
8766 if (is_mcq_supported(hba) && !hba->scsi_host_added) {
8767 ufshcd_mcq_enable(hba);
8768 ret = ufshcd_alloc_mcq(hba);
8769 if (!ret) {
8770 ufshcd_config_mcq(hba);
8771 } else {
8772 /* Continue with SDB mode */
8773 ufshcd_mcq_disable(hba);
8774 use_mcq_mode = false;
8775 dev_err(hba->dev, "MCQ mode is disabled, err=%d\n",
8776 ret);
8777 }
8778 ret = scsi_add_host(host, hba->dev);
8779 if (ret) {
8780 dev_err(hba->dev, "scsi_add_host failed\n");
8781 return ret;
8782 }
8783 hba->scsi_host_added = true;
8784 } else if (is_mcq_supported(hba)) {
8785 /* UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH is set */
8786 ufshcd_config_mcq(hba);
8787 ufshcd_mcq_enable(hba);
8788 }
8789 }
8790
8791 ufshcd_tune_unipro_params(hba);
8792
8793 /* UFS device is also active now */
8794 ufshcd_set_ufs_dev_active(hba);
8795 ufshcd_force_reset_auto_bkops(hba);
8796
8797 ufshcd_set_timestamp_attr(hba);
8798 schedule_delayed_work(&hba->ufs_rtc_update_work,
8799 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS));
8800
8801 /* Gear up to HS gear if supported */
8802 if (hba->max_pwr_info.is_valid) {
8803 /*
8804 * Set the right value to bRefClkFreq before attempting to
8805 * switch to HS gears.
8806 */
8807 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
8808 ufshcd_set_dev_ref_clk(hba);
8809 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8810 if (ret) {
8811 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
8812 __func__, ret);
8813 return ret;
8814 }
8815 }
8816
8817 return 0;
8818 }
8819
8820 /**
8821 * ufshcd_probe_hba - probe hba to detect device and initialize it
8822 * @hba: per-adapter instance
8823 * @init_dev_params: whether or not to call ufshcd_device_params_init().
8824 *
8825 * Execute link-startup and verify device initialization
8826 *
8827 * Return: 0 upon success; < 0 upon failure.
8828 */
ufshcd_probe_hba(struct ufs_hba * hba,bool init_dev_params)8829 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params)
8830 {
8831 ktime_t start = ktime_get();
8832 unsigned long flags;
8833 int ret;
8834
8835 ret = ufshcd_device_init(hba, init_dev_params);
8836 if (ret)
8837 goto out;
8838
8839 if (!hba->pm_op_in_progress &&
8840 (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) {
8841 /* Reset the device and controller before doing reinit */
8842 ufshcd_device_reset(hba);
8843 ufs_put_device_desc(hba);
8844 ufshcd_hba_stop(hba);
8845 ufshcd_vops_reinit_notify(hba);
8846 ret = ufshcd_hba_enable(hba);
8847 if (ret) {
8848 dev_err(hba->dev, "Host controller enable failed\n");
8849 ufshcd_print_evt_hist(hba);
8850 ufshcd_print_host_state(hba);
8851 goto out;
8852 }
8853
8854 /* Reinit the device */
8855 ret = ufshcd_device_init(hba, init_dev_params);
8856 if (ret)
8857 goto out;
8858 }
8859
8860 ufshcd_print_pwr_info(hba);
8861
8862 /*
8863 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
8864 * and for removable UFS card as well, hence always set the parameter.
8865 * Note: Error handler may issue the device reset hence resetting
8866 * bActiveICCLevel as well so it is always safe to set this here.
8867 */
8868 ufshcd_set_active_icc_lvl(hba);
8869
8870 /* Enable UFS Write Booster if supported */
8871 ufshcd_configure_wb(hba);
8872
8873 if (hba->ee_usr_mask)
8874 ufshcd_write_ee_control(hba);
8875 ufshcd_configure_auto_hibern8(hba);
8876
8877 out:
8878 spin_lock_irqsave(hba->host->host_lock, flags);
8879 if (ret)
8880 hba->ufshcd_state = UFSHCD_STATE_ERROR;
8881 else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
8882 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
8883 spin_unlock_irqrestore(hba->host->host_lock, flags);
8884
8885 trace_ufshcd_init(dev_name(hba->dev), ret,
8886 ktime_to_us(ktime_sub(ktime_get(), start)),
8887 hba->curr_dev_pwr_mode, hba->uic_link_state);
8888 return ret;
8889 }
8890
8891 /**
8892 * ufshcd_async_scan - asynchronous execution for probing hba
8893 * @data: data pointer to pass to this function
8894 * @cookie: cookie data
8895 */
ufshcd_async_scan(void * data,async_cookie_t cookie)8896 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
8897 {
8898 struct ufs_hba *hba = (struct ufs_hba *)data;
8899 int ret;
8900
8901 down(&hba->host_sem);
8902 /* Initialize hba, detect and initialize UFS device */
8903 ret = ufshcd_probe_hba(hba, true);
8904 up(&hba->host_sem);
8905 if (ret)
8906 goto out;
8907
8908 /* Probe and add UFS logical units */
8909 ret = ufshcd_add_lus(hba);
8910
8911 out:
8912 pm_runtime_put_sync(hba->dev);
8913
8914 if (ret)
8915 dev_err(hba->dev, "%s failed: %d\n", __func__, ret);
8916 }
8917
ufshcd_eh_timed_out(struct scsi_cmnd * scmd)8918 static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
8919 {
8920 struct ufs_hba *hba = shost_priv(scmd->device->host);
8921
8922 if (!hba->system_suspending) {
8923 /* Activate the error handler in the SCSI core. */
8924 return SCSI_EH_NOT_HANDLED;
8925 }
8926
8927 /*
8928 * If we get here we know that no TMFs are outstanding and also that
8929 * the only pending command is a START STOP UNIT command. Handle the
8930 * timeout of that command directly to prevent a deadlock between
8931 * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler().
8932 */
8933 ufshcd_link_recovery(hba);
8934 dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n",
8935 __func__, hba->outstanding_tasks);
8936
8937 return hba->outstanding_reqs ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE;
8938 }
8939
8940 static const struct attribute_group *ufshcd_driver_groups[] = {
8941 &ufs_sysfs_unit_descriptor_group,
8942 &ufs_sysfs_lun_attributes_group,
8943 NULL,
8944 };
8945
8946 static struct ufs_hba_variant_params ufs_hba_vps = {
8947 .hba_enable_delay_us = 1000,
8948 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40),
8949 .devfreq_profile.polling_ms = 100,
8950 .devfreq_profile.target = ufshcd_devfreq_target,
8951 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
8952 .ondemand_data.upthreshold = 70,
8953 .ondemand_data.downdifferential = 5,
8954 };
8955
8956 static const struct scsi_host_template ufshcd_driver_template = {
8957 .module = THIS_MODULE,
8958 .name = UFSHCD,
8959 .proc_name = UFSHCD,
8960 .map_queues = ufshcd_map_queues,
8961 .queuecommand = ufshcd_queuecommand,
8962 .mq_poll = ufshcd_poll,
8963 .slave_alloc = ufshcd_slave_alloc,
8964 .device_configure = ufshcd_device_configure,
8965 .slave_destroy = ufshcd_slave_destroy,
8966 .change_queue_depth = ufshcd_change_queue_depth,
8967 .eh_abort_handler = ufshcd_abort,
8968 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
8969 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
8970 .eh_timed_out = ufshcd_eh_timed_out,
8971 .this_id = -1,
8972 .sg_tablesize = SG_ALL,
8973 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
8974 .max_sectors = SZ_1M / SECTOR_SIZE,
8975 .max_host_blocked = 1,
8976 .track_queue_depth = 1,
8977 .skip_settle_delay = 1,
8978 .sdev_groups = ufshcd_driver_groups,
8979 };
8980
ufshcd_config_vreg_load(struct device * dev,struct ufs_vreg * vreg,int ua)8981 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
8982 int ua)
8983 {
8984 int ret;
8985
8986 if (!vreg)
8987 return 0;
8988
8989 /*
8990 * "set_load" operation shall be required on those regulators
8991 * which specifically configured current limitation. Otherwise
8992 * zero max_uA may cause unexpected behavior when regulator is
8993 * enabled or set as high power mode.
8994 */
8995 if (!vreg->max_uA)
8996 return 0;
8997
8998 ret = regulator_set_load(vreg->reg, ua);
8999 if (ret < 0) {
9000 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
9001 __func__, vreg->name, ua, ret);
9002 }
9003
9004 return ret;
9005 }
9006
ufshcd_config_vreg_lpm(struct ufs_hba * hba,struct ufs_vreg * vreg)9007 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
9008 struct ufs_vreg *vreg)
9009 {
9010 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
9011 }
9012
ufshcd_config_vreg_hpm(struct ufs_hba * hba,struct ufs_vreg * vreg)9013 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
9014 struct ufs_vreg *vreg)
9015 {
9016 if (!vreg)
9017 return 0;
9018
9019 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
9020 }
9021
ufshcd_config_vreg(struct device * dev,struct ufs_vreg * vreg,bool on)9022 static int ufshcd_config_vreg(struct device *dev,
9023 struct ufs_vreg *vreg, bool on)
9024 {
9025 if (regulator_count_voltages(vreg->reg) <= 0)
9026 return 0;
9027
9028 return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0);
9029 }
9030
ufshcd_enable_vreg(struct device * dev,struct ufs_vreg * vreg)9031 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
9032 {
9033 int ret = 0;
9034
9035 if (!vreg || vreg->enabled)
9036 goto out;
9037
9038 ret = ufshcd_config_vreg(dev, vreg, true);
9039 if (!ret)
9040 ret = regulator_enable(vreg->reg);
9041
9042 if (!ret)
9043 vreg->enabled = true;
9044 else
9045 dev_err(dev, "%s: %s enable failed, err=%d\n",
9046 __func__, vreg->name, ret);
9047 out:
9048 return ret;
9049 }
9050
ufshcd_disable_vreg(struct device * dev,struct ufs_vreg * vreg)9051 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
9052 {
9053 int ret = 0;
9054
9055 if (!vreg || !vreg->enabled || vreg->always_on)
9056 goto out;
9057
9058 ret = regulator_disable(vreg->reg);
9059
9060 if (!ret) {
9061 /* ignore errors on applying disable config */
9062 ufshcd_config_vreg(dev, vreg, false);
9063 vreg->enabled = false;
9064 } else {
9065 dev_err(dev, "%s: %s disable failed, err=%d\n",
9066 __func__, vreg->name, ret);
9067 }
9068 out:
9069 return ret;
9070 }
9071
ufshcd_setup_vreg(struct ufs_hba * hba,bool on)9072 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
9073 {
9074 int ret = 0;
9075 struct device *dev = hba->dev;
9076 struct ufs_vreg_info *info = &hba->vreg_info;
9077
9078 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
9079 if (ret)
9080 goto out;
9081
9082 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
9083 if (ret)
9084 goto out;
9085
9086 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
9087
9088 out:
9089 if (ret) {
9090 ufshcd_toggle_vreg(dev, info->vccq2, false);
9091 ufshcd_toggle_vreg(dev, info->vccq, false);
9092 ufshcd_toggle_vreg(dev, info->vcc, false);
9093 }
9094 return ret;
9095 }
9096
ufshcd_setup_hba_vreg(struct ufs_hba * hba,bool on)9097 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
9098 {
9099 struct ufs_vreg_info *info = &hba->vreg_info;
9100
9101 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
9102 }
9103
ufshcd_get_vreg(struct device * dev,struct ufs_vreg * vreg)9104 int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
9105 {
9106 int ret = 0;
9107
9108 if (!vreg)
9109 goto out;
9110
9111 vreg->reg = devm_regulator_get(dev, vreg->name);
9112 if (IS_ERR(vreg->reg)) {
9113 ret = PTR_ERR(vreg->reg);
9114 dev_err(dev, "%s: %s get failed, err=%d\n",
9115 __func__, vreg->name, ret);
9116 }
9117 out:
9118 return ret;
9119 }
9120 EXPORT_SYMBOL_GPL(ufshcd_get_vreg);
9121
ufshcd_init_vreg(struct ufs_hba * hba)9122 static int ufshcd_init_vreg(struct ufs_hba *hba)
9123 {
9124 int ret = 0;
9125 struct device *dev = hba->dev;
9126 struct ufs_vreg_info *info = &hba->vreg_info;
9127
9128 ret = ufshcd_get_vreg(dev, info->vcc);
9129 if (ret)
9130 goto out;
9131
9132 ret = ufshcd_get_vreg(dev, info->vccq);
9133 if (!ret)
9134 ret = ufshcd_get_vreg(dev, info->vccq2);
9135 out:
9136 return ret;
9137 }
9138
ufshcd_init_hba_vreg(struct ufs_hba * hba)9139 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
9140 {
9141 struct ufs_vreg_info *info = &hba->vreg_info;
9142
9143 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
9144 }
9145
ufshcd_setup_clocks(struct ufs_hba * hba,bool on)9146 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
9147 {
9148 int ret = 0;
9149 struct ufs_clk_info *clki;
9150 struct list_head *head = &hba->clk_list_head;
9151 unsigned long flags;
9152 ktime_t start = ktime_get();
9153 bool clk_state_changed = false;
9154
9155 if (list_empty(head))
9156 goto out;
9157
9158 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
9159 if (ret)
9160 return ret;
9161
9162 list_for_each_entry(clki, head, list) {
9163 if (!IS_ERR_OR_NULL(clki->clk)) {
9164 /*
9165 * Don't disable clocks which are needed
9166 * to keep the link active.
9167 */
9168 if (ufshcd_is_link_active(hba) &&
9169 clki->keep_link_active)
9170 continue;
9171
9172 clk_state_changed = on ^ clki->enabled;
9173 if (on && !clki->enabled) {
9174 ret = clk_prepare_enable(clki->clk);
9175 if (ret) {
9176 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
9177 __func__, clki->name, ret);
9178 goto out;
9179 }
9180 } else if (!on && clki->enabled) {
9181 clk_disable_unprepare(clki->clk);
9182 }
9183 clki->enabled = on;
9184 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
9185 clki->name, on ? "en" : "dis");
9186 }
9187 }
9188
9189 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
9190 if (ret)
9191 return ret;
9192
9193 if (!ufshcd_is_clkscaling_supported(hba))
9194 ufshcd_pm_qos_update(hba, on);
9195 out:
9196 if (ret) {
9197 list_for_each_entry(clki, head, list) {
9198 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
9199 clk_disable_unprepare(clki->clk);
9200 }
9201 } else if (!ret && on) {
9202 spin_lock_irqsave(hba->host->host_lock, flags);
9203 hba->clk_gating.state = CLKS_ON;
9204 trace_ufshcd_clk_gating(dev_name(hba->dev),
9205 hba->clk_gating.state);
9206 spin_unlock_irqrestore(hba->host->host_lock, flags);
9207 }
9208
9209 if (clk_state_changed)
9210 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
9211 (on ? "on" : "off"),
9212 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
9213 return ret;
9214 }
9215
ufshcd_parse_ref_clk_property(struct ufs_hba * hba)9216 static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba)
9217 {
9218 u32 freq;
9219 int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq);
9220
9221 if (ret) {
9222 dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret);
9223 return REF_CLK_FREQ_INVAL;
9224 }
9225
9226 return ufs_get_bref_clk_from_hz(freq);
9227 }
9228
ufshcd_init_clocks(struct ufs_hba * hba)9229 static int ufshcd_init_clocks(struct ufs_hba *hba)
9230 {
9231 int ret = 0;
9232 struct ufs_clk_info *clki;
9233 struct device *dev = hba->dev;
9234 struct list_head *head = &hba->clk_list_head;
9235
9236 if (list_empty(head))
9237 goto out;
9238
9239 list_for_each_entry(clki, head, list) {
9240 if (!clki->name)
9241 continue;
9242
9243 clki->clk = devm_clk_get(dev, clki->name);
9244 if (IS_ERR(clki->clk)) {
9245 ret = PTR_ERR(clki->clk);
9246 dev_err(dev, "%s: %s clk get failed, %d\n",
9247 __func__, clki->name, ret);
9248 goto out;
9249 }
9250
9251 /*
9252 * Parse device ref clk freq as per device tree "ref_clk".
9253 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
9254 * in ufshcd_alloc_host().
9255 */
9256 if (!strcmp(clki->name, "ref_clk"))
9257 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
9258
9259 if (clki->max_freq) {
9260 ret = clk_set_rate(clki->clk, clki->max_freq);
9261 if (ret) {
9262 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
9263 __func__, clki->name,
9264 clki->max_freq, ret);
9265 goto out;
9266 }
9267 clki->curr_freq = clki->max_freq;
9268 }
9269 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
9270 clki->name, clk_get_rate(clki->clk));
9271 }
9272
9273 /* Set Max. frequency for all clocks */
9274 if (hba->use_pm_opp) {
9275 ret = ufshcd_opp_set_rate(hba, ULONG_MAX);
9276 if (ret) {
9277 dev_err(hba->dev, "%s: failed to set OPP: %d", __func__,
9278 ret);
9279 goto out;
9280 }
9281 }
9282
9283 out:
9284 return ret;
9285 }
9286
ufshcd_variant_hba_init(struct ufs_hba * hba)9287 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
9288 {
9289 int err = 0;
9290
9291 if (!hba->vops)
9292 goto out;
9293
9294 err = ufshcd_vops_init(hba);
9295 if (err)
9296 dev_err_probe(hba->dev, err,
9297 "%s: variant %s init failed with err %d\n",
9298 __func__, ufshcd_get_var_name(hba), err);
9299 out:
9300 return err;
9301 }
9302
ufshcd_variant_hba_exit(struct ufs_hba * hba)9303 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
9304 {
9305 if (!hba->vops)
9306 return;
9307
9308 ufshcd_vops_exit(hba);
9309 }
9310
ufshcd_hba_init(struct ufs_hba * hba)9311 static int ufshcd_hba_init(struct ufs_hba *hba)
9312 {
9313 int err;
9314
9315 /*
9316 * Handle host controller power separately from the UFS device power
9317 * rails as it will help controlling the UFS host controller power
9318 * collapse easily which is different than UFS device power collapse.
9319 * Also, enable the host controller power before we go ahead with rest
9320 * of the initialization here.
9321 */
9322 err = ufshcd_init_hba_vreg(hba);
9323 if (err)
9324 goto out;
9325
9326 err = ufshcd_setup_hba_vreg(hba, true);
9327 if (err)
9328 goto out;
9329
9330 err = ufshcd_init_clocks(hba);
9331 if (err)
9332 goto out_disable_hba_vreg;
9333
9334 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
9335 hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba);
9336
9337 err = ufshcd_setup_clocks(hba, true);
9338 if (err)
9339 goto out_disable_hba_vreg;
9340
9341 err = ufshcd_init_vreg(hba);
9342 if (err)
9343 goto out_disable_clks;
9344
9345 err = ufshcd_setup_vreg(hba, true);
9346 if (err)
9347 goto out_disable_clks;
9348
9349 err = ufshcd_variant_hba_init(hba);
9350 if (err)
9351 goto out_disable_vreg;
9352
9353 ufs_debugfs_hba_init(hba);
9354 ufs_fault_inject_hba_init(hba);
9355
9356 hba->is_powered = true;
9357 goto out;
9358
9359 out_disable_vreg:
9360 ufshcd_setup_vreg(hba, false);
9361 out_disable_clks:
9362 ufshcd_setup_clocks(hba, false);
9363 out_disable_hba_vreg:
9364 ufshcd_setup_hba_vreg(hba, false);
9365 out:
9366 return err;
9367 }
9368
ufshcd_hba_exit(struct ufs_hba * hba)9369 static void ufshcd_hba_exit(struct ufs_hba *hba)
9370 {
9371 if (hba->is_powered) {
9372 ufshcd_pm_qos_exit(hba);
9373 ufshcd_exit_clk_scaling(hba);
9374 ufshcd_exit_clk_gating(hba);
9375 if (hba->eh_wq)
9376 destroy_workqueue(hba->eh_wq);
9377 ufs_debugfs_hba_exit(hba);
9378 ufshcd_variant_hba_exit(hba);
9379 ufshcd_setup_vreg(hba, false);
9380 ufshcd_setup_clocks(hba, false);
9381 ufshcd_setup_hba_vreg(hba, false);
9382 hba->is_powered = false;
9383 ufs_put_device_desc(hba);
9384 }
9385 }
9386
ufshcd_execute_start_stop(struct scsi_device * sdev,enum ufs_dev_pwr_mode pwr_mode,struct scsi_sense_hdr * sshdr)9387 static int ufshcd_execute_start_stop(struct scsi_device *sdev,
9388 enum ufs_dev_pwr_mode pwr_mode,
9389 struct scsi_sense_hdr *sshdr)
9390 {
9391 const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 };
9392 struct scsi_failure failure_defs[] = {
9393 {
9394 .allowed = 2,
9395 .result = SCMD_FAILURE_RESULT_ANY,
9396 },
9397 };
9398 struct scsi_failures failures = {
9399 .failure_definitions = failure_defs,
9400 };
9401 const struct scsi_exec_args args = {
9402 .failures = &failures,
9403 .sshdr = sshdr,
9404 .req_flags = BLK_MQ_REQ_PM,
9405 .scmd_flags = SCMD_FAIL_IF_RECOVERING,
9406 };
9407
9408 return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL,
9409 /*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0,
9410 &args);
9411 }
9412
9413 /**
9414 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
9415 * power mode
9416 * @hba: per adapter instance
9417 * @pwr_mode: device power mode to set
9418 *
9419 * Return: 0 if requested power mode is set successfully;
9420 * < 0 if failed to set the requested power mode.
9421 */
ufshcd_set_dev_pwr_mode(struct ufs_hba * hba,enum ufs_dev_pwr_mode pwr_mode)9422 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
9423 enum ufs_dev_pwr_mode pwr_mode)
9424 {
9425 struct scsi_sense_hdr sshdr;
9426 struct scsi_device *sdp;
9427 unsigned long flags;
9428 int ret;
9429
9430 spin_lock_irqsave(hba->host->host_lock, flags);
9431 sdp = hba->ufs_device_wlun;
9432 if (sdp && scsi_device_online(sdp))
9433 ret = scsi_device_get(sdp);
9434 else
9435 ret = -ENODEV;
9436 spin_unlock_irqrestore(hba->host->host_lock, flags);
9437
9438 if (ret)
9439 return ret;
9440
9441 /*
9442 * If scsi commands fail, the scsi mid-layer schedules scsi error-
9443 * handling, which would wait for host to be resumed. Since we know
9444 * we are functional while we are here, skip host resume in error
9445 * handling context.
9446 */
9447 hba->host->eh_noresume = 1;
9448
9449 /*
9450 * Current function would be generally called from the power management
9451 * callbacks hence set the RQF_PM flag so that it doesn't resume the
9452 * already suspended childs.
9453 */
9454 ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr);
9455 if (ret) {
9456 sdev_printk(KERN_WARNING, sdp,
9457 "START_STOP failed for power mode: %d, result %x\n",
9458 pwr_mode, ret);
9459 if (ret > 0) {
9460 if (scsi_sense_valid(&sshdr))
9461 scsi_print_sense_hdr(sdp, NULL, &sshdr);
9462 ret = -EIO;
9463 }
9464 } else {
9465 hba->curr_dev_pwr_mode = pwr_mode;
9466 }
9467
9468 scsi_device_put(sdp);
9469 hba->host->eh_noresume = 0;
9470 return ret;
9471 }
9472
ufshcd_link_state_transition(struct ufs_hba * hba,enum uic_link_state req_link_state,bool check_for_bkops)9473 static int ufshcd_link_state_transition(struct ufs_hba *hba,
9474 enum uic_link_state req_link_state,
9475 bool check_for_bkops)
9476 {
9477 int ret = 0;
9478
9479 if (req_link_state == hba->uic_link_state)
9480 return 0;
9481
9482 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
9483 ret = ufshcd_uic_hibern8_enter(hba);
9484 if (!ret) {
9485 ufshcd_set_link_hibern8(hba);
9486 } else {
9487 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9488 __func__, ret);
9489 goto out;
9490 }
9491 }
9492 /*
9493 * If autobkops is enabled, link can't be turned off because
9494 * turning off the link would also turn off the device, except in the
9495 * case of DeepSleep where the device is expected to remain powered.
9496 */
9497 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
9498 (!check_for_bkops || !hba->auto_bkops_enabled)) {
9499 /*
9500 * Let's make sure that link is in low power mode, we are doing
9501 * this currently by putting the link in Hibern8. Otherway to
9502 * put the link in low power mode is to send the DME end point
9503 * to device and then send the DME reset command to local
9504 * unipro. But putting the link in hibern8 is much faster.
9505 *
9506 * Note also that putting the link in Hibern8 is a requirement
9507 * for entering DeepSleep.
9508 */
9509 ret = ufshcd_uic_hibern8_enter(hba);
9510 if (ret) {
9511 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9512 __func__, ret);
9513 goto out;
9514 }
9515 /*
9516 * Change controller state to "reset state" which
9517 * should also put the link in off/reset state
9518 */
9519 ufshcd_hba_stop(hba);
9520 /*
9521 * TODO: Check if we need any delay to make sure that
9522 * controller is reset
9523 */
9524 ufshcd_set_link_off(hba);
9525 }
9526
9527 out:
9528 return ret;
9529 }
9530
ufshcd_vreg_set_lpm(struct ufs_hba * hba)9531 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
9532 {
9533 bool vcc_off = false;
9534
9535 /*
9536 * It seems some UFS devices may keep drawing more than sleep current
9537 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
9538 * To avoid this situation, add 2ms delay before putting these UFS
9539 * rails in LPM mode.
9540 */
9541 if (!ufshcd_is_link_active(hba) &&
9542 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
9543 usleep_range(2000, 2100);
9544
9545 /*
9546 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
9547 * power.
9548 *
9549 * If UFS device and link is in OFF state, all power supplies (VCC,
9550 * VCCQ, VCCQ2) can be turned off if power on write protect is not
9551 * required. If UFS link is inactive (Hibern8 or OFF state) and device
9552 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
9553 *
9554 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
9555 * in low power state which would save some power.
9556 *
9557 * If Write Booster is enabled and the device needs to flush the WB
9558 * buffer OR if bkops status is urgent for WB, keep Vcc on.
9559 */
9560 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
9561 !hba->dev_info.is_lu_power_on_wp) {
9562 ufshcd_setup_vreg(hba, false);
9563 vcc_off = true;
9564 } else if (!ufshcd_is_ufs_dev_active(hba)) {
9565 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9566 vcc_off = true;
9567 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) {
9568 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9569 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
9570 }
9571 }
9572
9573 /*
9574 * Some UFS devices require delay after VCC power rail is turned-off.
9575 */
9576 if (vcc_off && hba->vreg_info.vcc &&
9577 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
9578 usleep_range(5000, 5100);
9579 }
9580
9581 #ifdef CONFIG_PM
ufshcd_vreg_set_hpm(struct ufs_hba * hba)9582 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
9583 {
9584 int ret = 0;
9585
9586 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
9587 !hba->dev_info.is_lu_power_on_wp) {
9588 ret = ufshcd_setup_vreg(hba, true);
9589 } else if (!ufshcd_is_ufs_dev_active(hba)) {
9590 if (!ufshcd_is_link_active(hba)) {
9591 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
9592 if (ret)
9593 goto vcc_disable;
9594 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
9595 if (ret)
9596 goto vccq_lpm;
9597 }
9598 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
9599 }
9600 goto out;
9601
9602 vccq_lpm:
9603 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9604 vcc_disable:
9605 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9606 out:
9607 return ret;
9608 }
9609 #endif /* CONFIG_PM */
9610
ufshcd_hba_vreg_set_lpm(struct ufs_hba * hba)9611 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
9612 {
9613 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
9614 ufshcd_setup_hba_vreg(hba, false);
9615 }
9616
ufshcd_hba_vreg_set_hpm(struct ufs_hba * hba)9617 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
9618 {
9619 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
9620 ufshcd_setup_hba_vreg(hba, true);
9621 }
9622
__ufshcd_wl_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op)9623 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9624 {
9625 int ret = 0;
9626 bool check_for_bkops;
9627 enum ufs_pm_level pm_lvl;
9628 enum ufs_dev_pwr_mode req_dev_pwr_mode;
9629 enum uic_link_state req_link_state;
9630
9631 hba->pm_op_in_progress = true;
9632 if (pm_op != UFS_SHUTDOWN_PM) {
9633 pm_lvl = pm_op == UFS_RUNTIME_PM ?
9634 hba->rpm_lvl : hba->spm_lvl;
9635 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
9636 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
9637 } else {
9638 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
9639 req_link_state = UIC_LINK_OFF_STATE;
9640 }
9641
9642 /*
9643 * If we can't transition into any of the low power modes
9644 * just gate the clocks.
9645 */
9646 ufshcd_hold(hba);
9647 hba->clk_gating.is_suspended = true;
9648
9649 if (ufshcd_is_clkscaling_supported(hba))
9650 ufshcd_clk_scaling_suspend(hba, true);
9651
9652 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
9653 req_link_state == UIC_LINK_ACTIVE_STATE) {
9654 goto vops_suspend;
9655 }
9656
9657 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
9658 (req_link_state == hba->uic_link_state))
9659 goto enable_scaling;
9660
9661 /* UFS device & link must be active before we enter in this function */
9662 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
9663 /* Wait err handler finish or trigger err recovery */
9664 if (!ufshcd_eh_in_progress(hba))
9665 ufshcd_force_error_recovery(hba);
9666 ret = -EBUSY;
9667 goto enable_scaling;
9668 }
9669
9670 if (pm_op == UFS_RUNTIME_PM) {
9671 if (ufshcd_can_autobkops_during_suspend(hba)) {
9672 /*
9673 * The device is idle with no requests in the queue,
9674 * allow background operations if bkops status shows
9675 * that performance might be impacted.
9676 */
9677 ret = ufshcd_bkops_ctrl(hba);
9678 if (ret) {
9679 /*
9680 * If return err in suspend flow, IO will hang.
9681 * Trigger error handler and break suspend for
9682 * error recovery.
9683 */
9684 ufshcd_force_error_recovery(hba);
9685 ret = -EBUSY;
9686 goto enable_scaling;
9687 }
9688 } else {
9689 /* make sure that auto bkops is disabled */
9690 ufshcd_disable_auto_bkops(hba);
9691 }
9692 /*
9693 * If device needs to do BKOP or WB buffer flush during
9694 * Hibern8, keep device power mode as "active power mode"
9695 * and VCC supply.
9696 */
9697 hba->dev_info.b_rpm_dev_flush_capable =
9698 hba->auto_bkops_enabled ||
9699 (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
9700 ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
9701 ufshcd_is_auto_hibern8_enabled(hba))) &&
9702 ufshcd_wb_need_flush(hba));
9703 }
9704
9705 flush_work(&hba->eeh_work);
9706
9707 ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9708 if (ret)
9709 goto enable_scaling;
9710
9711 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
9712 if (pm_op != UFS_RUNTIME_PM)
9713 /* ensure that bkops is disabled */
9714 ufshcd_disable_auto_bkops(hba);
9715
9716 if (!hba->dev_info.b_rpm_dev_flush_capable) {
9717 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
9718 if (ret && pm_op != UFS_SHUTDOWN_PM) {
9719 /*
9720 * If return err in suspend flow, IO will hang.
9721 * Trigger error handler and break suspend for
9722 * error recovery.
9723 */
9724 ufshcd_force_error_recovery(hba);
9725 ret = -EBUSY;
9726 }
9727 if (ret)
9728 goto enable_scaling;
9729 }
9730 }
9731
9732 /*
9733 * In the case of DeepSleep, the device is expected to remain powered
9734 * with the link off, so do not check for bkops.
9735 */
9736 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
9737 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
9738 if (ret && pm_op != UFS_SHUTDOWN_PM) {
9739 /*
9740 * If return err in suspend flow, IO will hang.
9741 * Trigger error handler and break suspend for
9742 * error recovery.
9743 */
9744 ufshcd_force_error_recovery(hba);
9745 ret = -EBUSY;
9746 }
9747 if (ret)
9748 goto set_dev_active;
9749
9750 vops_suspend:
9751 /*
9752 * Call vendor specific suspend callback. As these callbacks may access
9753 * vendor specific host controller register space call them before the
9754 * host clocks are ON.
9755 */
9756 ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9757 if (ret)
9758 goto set_link_active;
9759
9760 cancel_delayed_work_sync(&hba->ufs_rtc_update_work);
9761 goto out;
9762
9763 set_link_active:
9764 /*
9765 * Device hardware reset is required to exit DeepSleep. Also, for
9766 * DeepSleep, the link is off so host reset and restore will be done
9767 * further below.
9768 */
9769 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9770 ufshcd_device_reset(hba);
9771 WARN_ON(!ufshcd_is_link_off(hba));
9772 }
9773 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
9774 ufshcd_set_link_active(hba);
9775 else if (ufshcd_is_link_off(hba))
9776 ufshcd_host_reset_and_restore(hba);
9777 set_dev_active:
9778 /* Can also get here needing to exit DeepSleep */
9779 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9780 ufshcd_device_reset(hba);
9781 ufshcd_host_reset_and_restore(hba);
9782 }
9783 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
9784 ufshcd_disable_auto_bkops(hba);
9785 enable_scaling:
9786 if (ufshcd_is_clkscaling_supported(hba))
9787 ufshcd_clk_scaling_suspend(hba, false);
9788
9789 hba->dev_info.b_rpm_dev_flush_capable = false;
9790 out:
9791 if (hba->dev_info.b_rpm_dev_flush_capable) {
9792 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
9793 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
9794 }
9795
9796 if (ret) {
9797 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret);
9798 hba->clk_gating.is_suspended = false;
9799 ufshcd_release(hba);
9800 }
9801 hba->pm_op_in_progress = false;
9802 return ret;
9803 }
9804
9805 #ifdef CONFIG_PM
__ufshcd_wl_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)9806 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9807 {
9808 int ret;
9809 enum uic_link_state old_link_state = hba->uic_link_state;
9810
9811 hba->pm_op_in_progress = true;
9812
9813 /*
9814 * Call vendor specific resume callback. As these callbacks may access
9815 * vendor specific host controller register space call them when the
9816 * host clocks are ON.
9817 */
9818 ret = ufshcd_vops_resume(hba, pm_op);
9819 if (ret)
9820 goto out;
9821
9822 /* For DeepSleep, the only supported option is to have the link off */
9823 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
9824
9825 if (ufshcd_is_link_hibern8(hba)) {
9826 ret = ufshcd_uic_hibern8_exit(hba);
9827 if (!ret) {
9828 ufshcd_set_link_active(hba);
9829 } else {
9830 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
9831 __func__, ret);
9832 goto vendor_suspend;
9833 }
9834 } else if (ufshcd_is_link_off(hba)) {
9835 /*
9836 * A full initialization of the host and the device is
9837 * required since the link was put to off during suspend.
9838 * Note, in the case of DeepSleep, the device will exit
9839 * DeepSleep due to device reset.
9840 */
9841 ret = ufshcd_reset_and_restore(hba);
9842 /*
9843 * ufshcd_reset_and_restore() should have already
9844 * set the link state as active
9845 */
9846 if (ret || !ufshcd_is_link_active(hba))
9847 goto vendor_suspend;
9848 }
9849
9850 if (!ufshcd_is_ufs_dev_active(hba)) {
9851 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
9852 if (ret)
9853 goto set_old_link_state;
9854 ufshcd_set_timestamp_attr(hba);
9855 schedule_delayed_work(&hba->ufs_rtc_update_work,
9856 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS));
9857 }
9858
9859 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
9860 ufshcd_enable_auto_bkops(hba);
9861 else
9862 /*
9863 * If BKOPs operations are urgently needed at this moment then
9864 * keep auto-bkops enabled or else disable it.
9865 */
9866 ufshcd_bkops_ctrl(hba);
9867
9868 if (hba->ee_usr_mask)
9869 ufshcd_write_ee_control(hba);
9870
9871 if (ufshcd_is_clkscaling_supported(hba))
9872 ufshcd_clk_scaling_suspend(hba, false);
9873
9874 if (hba->dev_info.b_rpm_dev_flush_capable) {
9875 hba->dev_info.b_rpm_dev_flush_capable = false;
9876 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
9877 }
9878
9879 ufshcd_configure_auto_hibern8(hba);
9880
9881 goto out;
9882
9883 set_old_link_state:
9884 ufshcd_link_state_transition(hba, old_link_state, 0);
9885 vendor_suspend:
9886 ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9887 ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9888 out:
9889 if (ret)
9890 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret);
9891 hba->clk_gating.is_suspended = false;
9892 ufshcd_release(hba);
9893 hba->pm_op_in_progress = false;
9894 return ret;
9895 }
9896
ufshcd_wl_runtime_suspend(struct device * dev)9897 static int ufshcd_wl_runtime_suspend(struct device *dev)
9898 {
9899 struct scsi_device *sdev = to_scsi_device(dev);
9900 struct ufs_hba *hba;
9901 int ret;
9902 ktime_t start = ktime_get();
9903
9904 hba = shost_priv(sdev->host);
9905
9906 ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM);
9907 if (ret)
9908 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9909
9910 trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret,
9911 ktime_to_us(ktime_sub(ktime_get(), start)),
9912 hba->curr_dev_pwr_mode, hba->uic_link_state);
9913
9914 return ret;
9915 }
9916
ufshcd_wl_runtime_resume(struct device * dev)9917 static int ufshcd_wl_runtime_resume(struct device *dev)
9918 {
9919 struct scsi_device *sdev = to_scsi_device(dev);
9920 struct ufs_hba *hba;
9921 int ret = 0;
9922 ktime_t start = ktime_get();
9923
9924 hba = shost_priv(sdev->host);
9925
9926 ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM);
9927 if (ret)
9928 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9929
9930 trace_ufshcd_wl_runtime_resume(dev_name(dev), ret,
9931 ktime_to_us(ktime_sub(ktime_get(), start)),
9932 hba->curr_dev_pwr_mode, hba->uic_link_state);
9933
9934 return ret;
9935 }
9936 #endif
9937
9938 #ifdef CONFIG_PM_SLEEP
ufshcd_wl_suspend(struct device * dev)9939 static int ufshcd_wl_suspend(struct device *dev)
9940 {
9941 struct scsi_device *sdev = to_scsi_device(dev);
9942 struct ufs_hba *hba;
9943 int ret = 0;
9944 ktime_t start = ktime_get();
9945
9946 hba = shost_priv(sdev->host);
9947 down(&hba->host_sem);
9948 hba->system_suspending = true;
9949
9950 if (pm_runtime_suspended(dev))
9951 goto out;
9952
9953 ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM);
9954 if (ret) {
9955 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9956 up(&hba->host_sem);
9957 }
9958
9959 out:
9960 if (!ret)
9961 hba->is_sys_suspended = true;
9962 trace_ufshcd_wl_suspend(dev_name(dev), ret,
9963 ktime_to_us(ktime_sub(ktime_get(), start)),
9964 hba->curr_dev_pwr_mode, hba->uic_link_state);
9965
9966 return ret;
9967 }
9968
ufshcd_wl_resume(struct device * dev)9969 static int ufshcd_wl_resume(struct device *dev)
9970 {
9971 struct scsi_device *sdev = to_scsi_device(dev);
9972 struct ufs_hba *hba;
9973 int ret = 0;
9974 ktime_t start = ktime_get();
9975
9976 hba = shost_priv(sdev->host);
9977
9978 if (pm_runtime_suspended(dev))
9979 goto out;
9980
9981 ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM);
9982 if (ret)
9983 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9984 out:
9985 trace_ufshcd_wl_resume(dev_name(dev), ret,
9986 ktime_to_us(ktime_sub(ktime_get(), start)),
9987 hba->curr_dev_pwr_mode, hba->uic_link_state);
9988 if (!ret)
9989 hba->is_sys_suspended = false;
9990 hba->system_suspending = false;
9991 up(&hba->host_sem);
9992 return ret;
9993 }
9994 #endif
9995
9996 /**
9997 * ufshcd_suspend - helper function for suspend operations
9998 * @hba: per adapter instance
9999 *
10000 * This function will put disable irqs, turn off clocks
10001 * and set vreg and hba-vreg in lpm mode.
10002 *
10003 * Return: 0 upon success; < 0 upon failure.
10004 */
ufshcd_suspend(struct ufs_hba * hba)10005 static int ufshcd_suspend(struct ufs_hba *hba)
10006 {
10007 int ret;
10008
10009 if (!hba->is_powered)
10010 return 0;
10011 /*
10012 * Disable the host irq as host controller as there won't be any
10013 * host controller transaction expected till resume.
10014 */
10015 ufshcd_disable_irq(hba);
10016 ret = ufshcd_setup_clocks(hba, false);
10017 if (ret) {
10018 ufshcd_enable_irq(hba);
10019 return ret;
10020 }
10021 if (ufshcd_is_clkgating_allowed(hba)) {
10022 hba->clk_gating.state = CLKS_OFF;
10023 trace_ufshcd_clk_gating(dev_name(hba->dev),
10024 hba->clk_gating.state);
10025 }
10026
10027 ufshcd_vreg_set_lpm(hba);
10028 /* Put the host controller in low power mode if possible */
10029 ufshcd_hba_vreg_set_lpm(hba);
10030 ufshcd_pm_qos_update(hba, false);
10031 return ret;
10032 }
10033
10034 #ifdef CONFIG_PM
10035 /**
10036 * ufshcd_resume - helper function for resume operations
10037 * @hba: per adapter instance
10038 *
10039 * This function basically turns on the regulators, clocks and
10040 * irqs of the hba.
10041 *
10042 * Return: 0 for success and non-zero for failure.
10043 */
ufshcd_resume(struct ufs_hba * hba)10044 static int ufshcd_resume(struct ufs_hba *hba)
10045 {
10046 int ret;
10047
10048 if (!hba->is_powered)
10049 return 0;
10050
10051 ufshcd_hba_vreg_set_hpm(hba);
10052 ret = ufshcd_vreg_set_hpm(hba);
10053 if (ret)
10054 goto out;
10055
10056 /* Make sure clocks are enabled before accessing controller */
10057 ret = ufshcd_setup_clocks(hba, true);
10058 if (ret)
10059 goto disable_vreg;
10060
10061 /* enable the host irq as host controller would be active soon */
10062 ufshcd_enable_irq(hba);
10063
10064 goto out;
10065
10066 disable_vreg:
10067 ufshcd_vreg_set_lpm(hba);
10068 out:
10069 if (ret)
10070 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
10071 return ret;
10072 }
10073 #endif /* CONFIG_PM */
10074
10075 #ifdef CONFIG_PM_SLEEP
10076 /**
10077 * ufshcd_system_suspend - system suspend callback
10078 * @dev: Device associated with the UFS controller.
10079 *
10080 * Executed before putting the system into a sleep state in which the contents
10081 * of main memory are preserved.
10082 *
10083 * Return: 0 for success and non-zero for failure.
10084 */
ufshcd_system_suspend(struct device * dev)10085 int ufshcd_system_suspend(struct device *dev)
10086 {
10087 struct ufs_hba *hba = dev_get_drvdata(dev);
10088 int ret = 0;
10089 ktime_t start = ktime_get();
10090
10091 if (pm_runtime_suspended(hba->dev))
10092 goto out;
10093
10094 ret = ufshcd_suspend(hba);
10095 out:
10096 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
10097 ktime_to_us(ktime_sub(ktime_get(), start)),
10098 hba->curr_dev_pwr_mode, hba->uic_link_state);
10099 return ret;
10100 }
10101 EXPORT_SYMBOL(ufshcd_system_suspend);
10102
10103 /**
10104 * ufshcd_system_resume - system resume callback
10105 * @dev: Device associated with the UFS controller.
10106 *
10107 * Executed after waking the system up from a sleep state in which the contents
10108 * of main memory were preserved.
10109 *
10110 * Return: 0 for success and non-zero for failure.
10111 */
ufshcd_system_resume(struct device * dev)10112 int ufshcd_system_resume(struct device *dev)
10113 {
10114 struct ufs_hba *hba = dev_get_drvdata(dev);
10115 ktime_t start = ktime_get();
10116 int ret = 0;
10117
10118 if (pm_runtime_suspended(hba->dev))
10119 goto out;
10120
10121 ret = ufshcd_resume(hba);
10122
10123 out:
10124 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
10125 ktime_to_us(ktime_sub(ktime_get(), start)),
10126 hba->curr_dev_pwr_mode, hba->uic_link_state);
10127
10128 return ret;
10129 }
10130 EXPORT_SYMBOL(ufshcd_system_resume);
10131 #endif /* CONFIG_PM_SLEEP */
10132
10133 #ifdef CONFIG_PM
10134 /**
10135 * ufshcd_runtime_suspend - runtime suspend callback
10136 * @dev: Device associated with the UFS controller.
10137 *
10138 * Check the description of ufshcd_suspend() function for more details.
10139 *
10140 * Return: 0 for success and non-zero for failure.
10141 */
ufshcd_runtime_suspend(struct device * dev)10142 int ufshcd_runtime_suspend(struct device *dev)
10143 {
10144 struct ufs_hba *hba = dev_get_drvdata(dev);
10145 int ret;
10146 ktime_t start = ktime_get();
10147
10148 ret = ufshcd_suspend(hba);
10149
10150 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
10151 ktime_to_us(ktime_sub(ktime_get(), start)),
10152 hba->curr_dev_pwr_mode, hba->uic_link_state);
10153 return ret;
10154 }
10155 EXPORT_SYMBOL(ufshcd_runtime_suspend);
10156
10157 /**
10158 * ufshcd_runtime_resume - runtime resume routine
10159 * @dev: Device associated with the UFS controller.
10160 *
10161 * This function basically brings controller
10162 * to active state. Following operations are done in this function:
10163 *
10164 * 1. Turn on all the controller related clocks
10165 * 2. Turn ON VCC rail
10166 *
10167 * Return: 0 upon success; < 0 upon failure.
10168 */
ufshcd_runtime_resume(struct device * dev)10169 int ufshcd_runtime_resume(struct device *dev)
10170 {
10171 struct ufs_hba *hba = dev_get_drvdata(dev);
10172 int ret;
10173 ktime_t start = ktime_get();
10174
10175 ret = ufshcd_resume(hba);
10176
10177 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
10178 ktime_to_us(ktime_sub(ktime_get(), start)),
10179 hba->curr_dev_pwr_mode, hba->uic_link_state);
10180 return ret;
10181 }
10182 EXPORT_SYMBOL(ufshcd_runtime_resume);
10183 #endif /* CONFIG_PM */
10184
ufshcd_wl_shutdown(struct device * dev)10185 static void ufshcd_wl_shutdown(struct device *dev)
10186 {
10187 struct scsi_device *sdev = to_scsi_device(dev);
10188 struct ufs_hba *hba = shost_priv(sdev->host);
10189
10190 down(&hba->host_sem);
10191 hba->shutting_down = true;
10192 up(&hba->host_sem);
10193
10194 /* Turn on everything while shutting down */
10195 ufshcd_rpm_get_sync(hba);
10196 scsi_device_quiesce(sdev);
10197 shost_for_each_device(sdev, hba->host) {
10198 if (sdev == hba->ufs_device_wlun)
10199 continue;
10200 mutex_lock(&sdev->state_mutex);
10201 scsi_device_set_state(sdev, SDEV_OFFLINE);
10202 mutex_unlock(&sdev->state_mutex);
10203 }
10204 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
10205
10206 /*
10207 * Next, turn off the UFS controller and the UFS regulators. Disable
10208 * clocks.
10209 */
10210 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
10211 ufshcd_suspend(hba);
10212
10213 hba->is_powered = false;
10214 }
10215
10216 /**
10217 * ufshcd_remove - de-allocate SCSI host and host memory space
10218 * data structure memory
10219 * @hba: per adapter instance
10220 */
ufshcd_remove(struct ufs_hba * hba)10221 void ufshcd_remove(struct ufs_hba *hba)
10222 {
10223 if (hba->ufs_device_wlun)
10224 ufshcd_rpm_get_sync(hba);
10225 ufs_hwmon_remove(hba);
10226 ufs_bsg_remove(hba);
10227 ufs_sysfs_remove_nodes(hba->dev);
10228 blk_mq_destroy_queue(hba->tmf_queue);
10229 blk_put_queue(hba->tmf_queue);
10230 blk_mq_free_tag_set(&hba->tmf_tag_set);
10231 if (hba->scsi_host_added)
10232 scsi_remove_host(hba->host);
10233 /* disable interrupts */
10234 ufshcd_disable_intr(hba, hba->intr_mask);
10235 ufshcd_hba_stop(hba);
10236 ufshcd_hba_exit(hba);
10237 }
10238 EXPORT_SYMBOL_GPL(ufshcd_remove);
10239
10240 #ifdef CONFIG_PM_SLEEP
ufshcd_system_freeze(struct device * dev)10241 int ufshcd_system_freeze(struct device *dev)
10242 {
10243
10244 return ufshcd_system_suspend(dev);
10245
10246 }
10247 EXPORT_SYMBOL_GPL(ufshcd_system_freeze);
10248
ufshcd_system_restore(struct device * dev)10249 int ufshcd_system_restore(struct device *dev)
10250 {
10251
10252 struct ufs_hba *hba = dev_get_drvdata(dev);
10253 int ret;
10254
10255 ret = ufshcd_system_resume(dev);
10256 if (ret)
10257 return ret;
10258
10259 /* Configure UTRL and UTMRL base address registers */
10260 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
10261 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
10262 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
10263 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
10264 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
10265 REG_UTP_TASK_REQ_LIST_BASE_L);
10266 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
10267 REG_UTP_TASK_REQ_LIST_BASE_H);
10268 /*
10269 * Make sure that UTRL and UTMRL base address registers
10270 * are updated with the latest queue addresses. Only after
10271 * updating these addresses, we can queue the new commands.
10272 */
10273 ufshcd_readl(hba, REG_UTP_TASK_REQ_LIST_BASE_H);
10274
10275 return 0;
10276
10277 }
10278 EXPORT_SYMBOL_GPL(ufshcd_system_restore);
10279
ufshcd_system_thaw(struct device * dev)10280 int ufshcd_system_thaw(struct device *dev)
10281 {
10282 return ufshcd_system_resume(dev);
10283 }
10284 EXPORT_SYMBOL_GPL(ufshcd_system_thaw);
10285 #endif /* CONFIG_PM_SLEEP */
10286
10287 /**
10288 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
10289 * @hba: pointer to Host Bus Adapter (HBA)
10290 */
ufshcd_dealloc_host(struct ufs_hba * hba)10291 void ufshcd_dealloc_host(struct ufs_hba *hba)
10292 {
10293 scsi_host_put(hba->host);
10294 }
10295 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
10296
10297 /**
10298 * ufshcd_set_dma_mask - Set dma mask based on the controller
10299 * addressing capability
10300 * @hba: per adapter instance
10301 *
10302 * Return: 0 for success, non-zero for failure.
10303 */
ufshcd_set_dma_mask(struct ufs_hba * hba)10304 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
10305 {
10306 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
10307 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
10308 return 0;
10309 }
10310 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
10311 }
10312
10313 /**
10314 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
10315 * @dev: pointer to device handle
10316 * @hba_handle: driver private handle
10317 *
10318 * Return: 0 on success, non-zero value on failure.
10319 */
ufshcd_alloc_host(struct device * dev,struct ufs_hba ** hba_handle)10320 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
10321 {
10322 struct Scsi_Host *host;
10323 struct ufs_hba *hba;
10324 int err = 0;
10325
10326 if (!dev) {
10327 dev_err(dev,
10328 "Invalid memory reference for dev is NULL\n");
10329 err = -ENODEV;
10330 goto out_error;
10331 }
10332
10333 host = scsi_host_alloc(&ufshcd_driver_template,
10334 sizeof(struct ufs_hba));
10335 if (!host) {
10336 dev_err(dev, "scsi_host_alloc failed\n");
10337 err = -ENOMEM;
10338 goto out_error;
10339 }
10340 host->nr_maps = HCTX_TYPE_POLL + 1;
10341 hba = shost_priv(host);
10342 hba->host = host;
10343 hba->dev = dev;
10344 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
10345 hba->nop_out_timeout = NOP_OUT_TIMEOUT;
10346 ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry));
10347 INIT_LIST_HEAD(&hba->clk_list_head);
10348 spin_lock_init(&hba->outstanding_lock);
10349
10350 *hba_handle = hba;
10351
10352 out_error:
10353 return err;
10354 }
10355 EXPORT_SYMBOL(ufshcd_alloc_host);
10356
10357 /* This function exists because blk_mq_alloc_tag_set() requires this. */
ufshcd_queue_tmf(struct blk_mq_hw_ctx * hctx,const struct blk_mq_queue_data * qd)10358 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
10359 const struct blk_mq_queue_data *qd)
10360 {
10361 WARN_ON_ONCE(true);
10362 return BLK_STS_NOTSUPP;
10363 }
10364
10365 static const struct blk_mq_ops ufshcd_tmf_ops = {
10366 .queue_rq = ufshcd_queue_tmf,
10367 };
10368
10369 /**
10370 * ufshcd_init - Driver initialization routine
10371 * @hba: per-adapter instance
10372 * @mmio_base: base register address
10373 * @irq: Interrupt line of device
10374 *
10375 * Return: 0 on success, non-zero value on failure.
10376 */
ufshcd_init(struct ufs_hba * hba,void __iomem * mmio_base,unsigned int irq)10377 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
10378 {
10379 int err;
10380 struct Scsi_Host *host = hba->host;
10381 struct device *dev = hba->dev;
10382
10383 /*
10384 * dev_set_drvdata() must be called before any callbacks are registered
10385 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
10386 * sysfs).
10387 */
10388 dev_set_drvdata(dev, hba);
10389
10390 if (!mmio_base) {
10391 dev_err(hba->dev,
10392 "Invalid memory reference for mmio_base is NULL\n");
10393 err = -ENODEV;
10394 goto out_error;
10395 }
10396
10397 hba->mmio_base = mmio_base;
10398 hba->irq = irq;
10399 hba->vps = &ufs_hba_vps;
10400
10401 err = ufshcd_hba_init(hba);
10402 if (err)
10403 goto out_error;
10404
10405 /* Read capabilities registers */
10406 err = ufshcd_hba_capabilities(hba);
10407 if (err)
10408 goto out_disable;
10409
10410 /* Get UFS version supported by the controller */
10411 hba->ufs_version = ufshcd_get_ufs_version(hba);
10412
10413 /* Get Interrupt bit mask per version */
10414 hba->intr_mask = ufshcd_get_intr_mask(hba);
10415
10416 err = ufshcd_set_dma_mask(hba);
10417 if (err) {
10418 dev_err(hba->dev, "set dma mask failed\n");
10419 goto out_disable;
10420 }
10421
10422 /* Allocate memory for host memory space */
10423 err = ufshcd_memory_alloc(hba);
10424 if (err) {
10425 dev_err(hba->dev, "Memory allocation failed\n");
10426 goto out_disable;
10427 }
10428
10429 /* Configure LRB */
10430 ufshcd_host_memory_configure(hba);
10431
10432 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
10433 host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
10434 host->max_id = UFSHCD_MAX_ID;
10435 host->max_lun = UFS_MAX_LUNS;
10436 host->max_channel = UFSHCD_MAX_CHANNEL;
10437 host->unique_id = host->host_no;
10438 host->max_cmd_len = UFS_CDB_SIZE;
10439 host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING);
10440
10441 /* Use default RPM delay if host not set */
10442 if (host->rpm_autosuspend_delay == 0)
10443 host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS;
10444
10445 hba->max_pwr_info.is_valid = false;
10446
10447 /* Initialize work queues */
10448 hba->eh_wq = alloc_ordered_workqueue("ufs_eh_wq_%d", WQ_MEM_RECLAIM,
10449 hba->host->host_no);
10450 if (!hba->eh_wq) {
10451 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
10452 __func__);
10453 err = -ENOMEM;
10454 goto out_disable;
10455 }
10456 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
10457 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
10458
10459 sema_init(&hba->host_sem, 1);
10460
10461 /* Initialize UIC command mutex */
10462 mutex_init(&hba->uic_cmd_mutex);
10463
10464 /* Initialize mutex for device management commands */
10465 mutex_init(&hba->dev_cmd.lock);
10466
10467 /* Initialize mutex for exception event control */
10468 mutex_init(&hba->ee_ctrl_mutex);
10469
10470 mutex_init(&hba->wb_mutex);
10471 init_rwsem(&hba->clk_scaling_lock);
10472
10473 ufshcd_init_clk_gating(hba);
10474
10475 ufshcd_init_clk_scaling(hba);
10476
10477 /*
10478 * In order to avoid any spurious interrupt immediately after
10479 * registering UFS controller interrupt handler, clear any pending UFS
10480 * interrupt status and disable all the UFS interrupts.
10481 */
10482 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
10483 REG_INTERRUPT_STATUS);
10484 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
10485 /*
10486 * Make sure that UFS interrupts are disabled and any pending interrupt
10487 * status is cleared before registering UFS interrupt handler.
10488 */
10489 ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
10490
10491 /* IRQ registration */
10492 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
10493 if (err) {
10494 dev_err(hba->dev, "request irq failed\n");
10495 goto out_disable;
10496 } else {
10497 hba->is_irq_enabled = true;
10498 }
10499
10500 if (!is_mcq_supported(hba)) {
10501 if (!hba->lsdb_sup) {
10502 dev_err(hba->dev, "%s: failed to initialize (legacy doorbell mode not supported)\n",
10503 __func__);
10504 err = -EINVAL;
10505 goto out_disable;
10506 }
10507 err = scsi_add_host(host, hba->dev);
10508 if (err) {
10509 dev_err(hba->dev, "scsi_add_host failed\n");
10510 goto out_disable;
10511 }
10512 hba->scsi_host_added = true;
10513 }
10514
10515 hba->tmf_tag_set = (struct blk_mq_tag_set) {
10516 .nr_hw_queues = 1,
10517 .queue_depth = hba->nutmrs,
10518 .ops = &ufshcd_tmf_ops,
10519 .flags = BLK_MQ_F_NO_SCHED,
10520 };
10521 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
10522 if (err < 0)
10523 goto out_remove_scsi_host;
10524 hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL);
10525 if (IS_ERR(hba->tmf_queue)) {
10526 err = PTR_ERR(hba->tmf_queue);
10527 goto free_tmf_tag_set;
10528 }
10529 hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs,
10530 sizeof(*hba->tmf_rqs), GFP_KERNEL);
10531 if (!hba->tmf_rqs) {
10532 err = -ENOMEM;
10533 goto free_tmf_queue;
10534 }
10535
10536 /* Reset the attached device */
10537 ufshcd_device_reset(hba);
10538
10539 ufshcd_init_crypto(hba);
10540
10541 /* Host controller enable */
10542 err = ufshcd_hba_enable(hba);
10543 if (err) {
10544 dev_err(hba->dev, "Host controller enable failed\n");
10545 ufshcd_print_evt_hist(hba);
10546 ufshcd_print_host_state(hba);
10547 goto free_tmf_queue;
10548 }
10549
10550 /*
10551 * Set the default power management level for runtime and system PM.
10552 * Default power saving mode is to keep UFS link in Hibern8 state
10553 * and UFS device in sleep state.
10554 */
10555 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10556 UFS_SLEEP_PWR_MODE,
10557 UIC_LINK_HIBERN8_STATE);
10558 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10559 UFS_SLEEP_PWR_MODE,
10560 UIC_LINK_HIBERN8_STATE);
10561
10562 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work);
10563 INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work);
10564
10565 /* Set the default auto-hiberate idle timer value to 150 ms */
10566 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
10567 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
10568 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
10569 }
10570
10571 /* Hold auto suspend until async scan completes */
10572 pm_runtime_get_sync(dev);
10573 atomic_set(&hba->scsi_block_reqs_cnt, 0);
10574 /*
10575 * We are assuming that device wasn't put in sleep/power-down
10576 * state exclusively during the boot stage before kernel.
10577 * This assumption helps avoid doing link startup twice during
10578 * ufshcd_probe_hba().
10579 */
10580 ufshcd_set_ufs_dev_active(hba);
10581
10582 async_schedule(ufshcd_async_scan, hba);
10583 ufs_sysfs_add_nodes(hba->dev);
10584
10585 device_enable_async_suspend(dev);
10586 ufshcd_pm_qos_init(hba);
10587 return 0;
10588
10589 free_tmf_queue:
10590 blk_mq_destroy_queue(hba->tmf_queue);
10591 blk_put_queue(hba->tmf_queue);
10592 free_tmf_tag_set:
10593 blk_mq_free_tag_set(&hba->tmf_tag_set);
10594 out_remove_scsi_host:
10595 if (hba->scsi_host_added)
10596 scsi_remove_host(hba->host);
10597 out_disable:
10598 hba->is_irq_enabled = false;
10599 ufshcd_hba_exit(hba);
10600 out_error:
10601 return err;
10602 }
10603 EXPORT_SYMBOL_GPL(ufshcd_init);
10604
ufshcd_resume_complete(struct device * dev)10605 void ufshcd_resume_complete(struct device *dev)
10606 {
10607 struct ufs_hba *hba = dev_get_drvdata(dev);
10608
10609 if (hba->complete_put) {
10610 ufshcd_rpm_put(hba);
10611 hba->complete_put = false;
10612 }
10613 }
10614 EXPORT_SYMBOL_GPL(ufshcd_resume_complete);
10615
ufshcd_rpm_ok_for_spm(struct ufs_hba * hba)10616 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba)
10617 {
10618 struct device *dev = &hba->ufs_device_wlun->sdev_gendev;
10619 enum ufs_dev_pwr_mode dev_pwr_mode;
10620 enum uic_link_state link_state;
10621 unsigned long flags;
10622 bool res;
10623
10624 spin_lock_irqsave(&dev->power.lock, flags);
10625 dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl);
10626 link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl);
10627 res = pm_runtime_suspended(dev) &&
10628 hba->curr_dev_pwr_mode == dev_pwr_mode &&
10629 hba->uic_link_state == link_state &&
10630 !hba->dev_info.b_rpm_dev_flush_capable;
10631 spin_unlock_irqrestore(&dev->power.lock, flags);
10632
10633 return res;
10634 }
10635
__ufshcd_suspend_prepare(struct device * dev,bool rpm_ok_for_spm)10636 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm)
10637 {
10638 struct ufs_hba *hba = dev_get_drvdata(dev);
10639 int ret;
10640
10641 /*
10642 * SCSI assumes that runtime-pm and system-pm for scsi drivers
10643 * are same. And it doesn't wake up the device for system-suspend
10644 * if it's runtime suspended. But ufs doesn't follow that.
10645 * Refer ufshcd_resume_complete()
10646 */
10647 if (hba->ufs_device_wlun) {
10648 /* Prevent runtime suspend */
10649 ufshcd_rpm_get_noresume(hba);
10650 /*
10651 * Check if already runtime suspended in same state as system
10652 * suspend would be.
10653 */
10654 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) {
10655 /* RPM state is not ok for SPM, so runtime resume */
10656 ret = ufshcd_rpm_resume(hba);
10657 if (ret < 0 && ret != -EACCES) {
10658 ufshcd_rpm_put(hba);
10659 return ret;
10660 }
10661 }
10662 hba->complete_put = true;
10663 }
10664 return 0;
10665 }
10666 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare);
10667
ufshcd_suspend_prepare(struct device * dev)10668 int ufshcd_suspend_prepare(struct device *dev)
10669 {
10670 return __ufshcd_suspend_prepare(dev, true);
10671 }
10672 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare);
10673
10674 #ifdef CONFIG_PM_SLEEP
ufshcd_wl_poweroff(struct device * dev)10675 static int ufshcd_wl_poweroff(struct device *dev)
10676 {
10677 struct scsi_device *sdev = to_scsi_device(dev);
10678 struct ufs_hba *hba = shost_priv(sdev->host);
10679
10680 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
10681 return 0;
10682 }
10683 #endif
10684
ufshcd_wl_probe(struct device * dev)10685 static int ufshcd_wl_probe(struct device *dev)
10686 {
10687 struct scsi_device *sdev = to_scsi_device(dev);
10688
10689 if (!is_device_wlun(sdev))
10690 return -ENODEV;
10691
10692 blk_pm_runtime_init(sdev->request_queue, dev);
10693 pm_runtime_set_autosuspend_delay(dev, 0);
10694 pm_runtime_allow(dev);
10695
10696 return 0;
10697 }
10698
ufshcd_wl_remove(struct device * dev)10699 static int ufshcd_wl_remove(struct device *dev)
10700 {
10701 pm_runtime_forbid(dev);
10702 return 0;
10703 }
10704
10705 static const struct dev_pm_ops ufshcd_wl_pm_ops = {
10706 #ifdef CONFIG_PM_SLEEP
10707 .suspend = ufshcd_wl_suspend,
10708 .resume = ufshcd_wl_resume,
10709 .freeze = ufshcd_wl_suspend,
10710 .thaw = ufshcd_wl_resume,
10711 .poweroff = ufshcd_wl_poweroff,
10712 .restore = ufshcd_wl_resume,
10713 #endif
10714 SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL)
10715 };
10716
ufshcd_check_header_layout(void)10717 static void ufshcd_check_header_layout(void)
10718 {
10719 /*
10720 * gcc compilers before version 10 cannot do constant-folding for
10721 * sub-byte bitfields. Hence skip the layout checks for gcc 9 and
10722 * before.
10723 */
10724 if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000)
10725 return;
10726
10727 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10728 .cci = 3})[0] != 3);
10729
10730 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10731 .ehs_length = 2})[1] != 2);
10732
10733 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10734 .enable_crypto = 1})[2]
10735 != 0x80);
10736
10737 BUILD_BUG_ON((((u8 *)&(struct request_desc_header){
10738 .command_type = 5,
10739 .data_direction = 3,
10740 .interrupt = 1,
10741 })[3]) != ((5 << 4) | (3 << 1) | 1));
10742
10743 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){
10744 .dunl = cpu_to_le32(0xdeadbeef)})[1] !=
10745 cpu_to_le32(0xdeadbeef));
10746
10747 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10748 .ocs = 4})[8] != 4);
10749
10750 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10751 .cds = 5})[9] != 5);
10752
10753 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){
10754 .dunu = cpu_to_le32(0xbadcafe)})[3] !=
10755 cpu_to_le32(0xbadcafe));
10756
10757 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){
10758 .iid = 0xf })[4] != 0xf0);
10759
10760 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){
10761 .command_set_type = 0xf })[4] != 0xf);
10762 }
10763
10764 /*
10765 * ufs_dev_wlun_template - describes ufs device wlun
10766 * ufs-device wlun - used to send pm commands
10767 * All luns are consumers of ufs-device wlun.
10768 *
10769 * Currently, no sd driver is present for wluns.
10770 * Hence the no specific pm operations are performed.
10771 * With ufs design, SSU should be sent to ufs-device wlun.
10772 * Hence register a scsi driver for ufs wluns only.
10773 */
10774 static struct scsi_driver ufs_dev_wlun_template = {
10775 .gendrv = {
10776 .name = "ufs_device_wlun",
10777 .probe = ufshcd_wl_probe,
10778 .remove = ufshcd_wl_remove,
10779 .pm = &ufshcd_wl_pm_ops,
10780 .shutdown = ufshcd_wl_shutdown,
10781 },
10782 };
10783
ufshcd_core_init(void)10784 static int __init ufshcd_core_init(void)
10785 {
10786 int ret;
10787
10788 ufshcd_check_header_layout();
10789
10790 ufs_debugfs_init();
10791
10792 ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
10793 if (ret)
10794 ufs_debugfs_exit();
10795 return ret;
10796 }
10797
ufshcd_core_exit(void)10798 static void __exit ufshcd_core_exit(void)
10799 {
10800 ufs_debugfs_exit();
10801 scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
10802 }
10803
10804 module_init(ufshcd_core_init);
10805 module_exit(ufshcd_core_exit);
10806
10807 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
10808 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
10809 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
10810 MODULE_SOFTDEP("pre: governor_simpleondemand");
10811 MODULE_LICENSE("GPL");
10812