1 /*
2 * Copyright (c) 2015-2018, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <errno.h>
9
10 #include <arch.h>
11 #include <arch_helpers.h>
12 #include <common/debug.h>
13 #include <drivers/delay_timer.h>
14 #include <denver.h>
15 #include <lib/mmio.h>
16 #include <plat/common/platform.h>
17
18 #include <mce_private.h>
19 #include <t18x_ari.h>
20
21 /*******************************************************************************
22 * Register offsets for ARI request/results
23 ******************************************************************************/
24 #define ARI_REQUEST 0x0U
25 #define ARI_REQUEST_EVENT_MASK 0x4U
26 #define ARI_STATUS 0x8U
27 #define ARI_REQUEST_DATA_LO 0xCU
28 #define ARI_REQUEST_DATA_HI 0x10U
29 #define ARI_RESPONSE_DATA_LO 0x14U
30 #define ARI_RESPONSE_DATA_HI 0x18U
31
32 /* Status values for the current request */
33 #define ARI_REQ_PENDING 1U
34 #define ARI_REQ_ONGOING 3U
35 #define ARI_REQUEST_VALID_BIT (1U << 8)
36 #define ARI_EVT_MASK_STANDBYWFI_BIT (1U << 7)
37
38 /* default timeout (us) to wait for ARI completion */
39 #define ARI_MAX_RETRY_COUNT U(2000000)
40
41 /*******************************************************************************
42 * ARI helper functions
43 ******************************************************************************/
ari_read_32(uint32_t ari_base,uint32_t reg)44 static inline uint32_t ari_read_32(uint32_t ari_base, uint32_t reg)
45 {
46 return mmio_read_32((uint64_t)ari_base + (uint64_t)reg);
47 }
48
ari_write_32(uint32_t ari_base,uint32_t val,uint32_t reg)49 static inline void ari_write_32(uint32_t ari_base, uint32_t val, uint32_t reg)
50 {
51 mmio_write_32((uint64_t)ari_base + (uint64_t)reg, val);
52 }
53
ari_get_request_low(uint32_t ari_base)54 static inline uint32_t ari_get_request_low(uint32_t ari_base)
55 {
56 return ari_read_32(ari_base, ARI_REQUEST_DATA_LO);
57 }
58
ari_get_request_high(uint32_t ari_base)59 static inline uint32_t ari_get_request_high(uint32_t ari_base)
60 {
61 return ari_read_32(ari_base, ARI_REQUEST_DATA_HI);
62 }
63
ari_get_response_low(uint32_t ari_base)64 static inline uint32_t ari_get_response_low(uint32_t ari_base)
65 {
66 return ari_read_32(ari_base, ARI_RESPONSE_DATA_LO);
67 }
68
ari_get_response_high(uint32_t ari_base)69 static inline uint32_t ari_get_response_high(uint32_t ari_base)
70 {
71 return ari_read_32(ari_base, ARI_RESPONSE_DATA_HI);
72 }
73
ari_clobber_response(uint32_t ari_base)74 static inline void ari_clobber_response(uint32_t ari_base)
75 {
76 ari_write_32(ari_base, 0, ARI_RESPONSE_DATA_LO);
77 ari_write_32(ari_base, 0, ARI_RESPONSE_DATA_HI);
78 }
79
ari_request_wait(uint32_t ari_base,uint32_t evt_mask,uint32_t req,uint32_t lo,uint32_t hi)80 static int32_t ari_request_wait(uint32_t ari_base, uint32_t evt_mask, uint32_t req,
81 uint32_t lo, uint32_t hi)
82 {
83 uint32_t retries = (uint32_t)ARI_MAX_RETRY_COUNT;
84 uint32_t status;
85 int32_t ret = 0;
86
87 /* program the request, event_mask, hi and lo registers */
88 ari_write_32(ari_base, lo, ARI_REQUEST_DATA_LO);
89 ari_write_32(ari_base, hi, ARI_REQUEST_DATA_HI);
90 ari_write_32(ari_base, evt_mask, ARI_REQUEST_EVENT_MASK);
91 ari_write_32(ari_base, req | ARI_REQUEST_VALID_BIT, ARI_REQUEST);
92
93 /*
94 * For commands that have an event trigger, we should bypass
95 * ARI_STATUS polling, since MCE is waiting for SW to trigger
96 * the event.
97 */
98 if (evt_mask != 0U) {
99 ret = 0;
100 } else {
101 /* For shutdown/reboot commands, we dont have to check for timeouts */
102 if ((req == TEGRA_ARI_MISC_CCPLEX) &&
103 ((lo == TEGRA_ARI_MISC_CCPLEX_SHUTDOWN_POWER_OFF) ||
104 (lo == TEGRA_ARI_MISC_CCPLEX_SHUTDOWN_REBOOT))) {
105 ret = 0;
106 } else {
107 /*
108 * Wait for the command response for not more than the timeout
109 */
110 while (retries != 0U) {
111
112 /* read the command status */
113 status = ari_read_32(ari_base, ARI_STATUS);
114 if ((status & (ARI_REQ_ONGOING | ARI_REQ_PENDING)) == 0U) {
115 break;
116 }
117
118 /* delay 1 us */
119 udelay(1);
120
121 /* decrement the retry count */
122 retries--;
123 }
124
125 /* assert if the command timed out */
126 if (retries == 0U) {
127 ERROR("ARI request timed out: req %d on CPU %d\n",
128 req, plat_my_core_pos());
129 assert(retries != 0U);
130 }
131 }
132 }
133
134 return ret;
135 }
136
ari_enter_cstate(uint32_t ari_base,uint32_t state,uint32_t wake_time)137 int32_t ari_enter_cstate(uint32_t ari_base, uint32_t state, uint32_t wake_time)
138 {
139 int32_t ret = 0;
140
141 /* check for allowed power state */
142 if ((state != TEGRA_ARI_CORE_C0) &&
143 (state != TEGRA_ARI_CORE_C1) &&
144 (state != TEGRA_ARI_CORE_C6) &&
145 (state != TEGRA_ARI_CORE_C7)) {
146 ERROR("%s: unknown cstate (%d)\n", __func__, state);
147 ret = EINVAL;
148 } else {
149 /* clean the previous response state */
150 ari_clobber_response(ari_base);
151
152 /* Enter the cstate, to be woken up after wake_time (TSC ticks) */
153 ret = ari_request_wait(ari_base, ARI_EVT_MASK_STANDBYWFI_BIT,
154 (uint32_t)TEGRA_ARI_ENTER_CSTATE, state, wake_time);
155 }
156
157 return ret;
158 }
159
ari_update_cstate_info(uint32_t ari_base,uint32_t cluster,uint32_t ccplex,uint32_t system,uint8_t sys_state_force,uint32_t wake_mask,uint8_t update_wake_mask)160 int32_t ari_update_cstate_info(uint32_t ari_base, uint32_t cluster, uint32_t ccplex,
161 uint32_t system, uint8_t sys_state_force, uint32_t wake_mask,
162 uint8_t update_wake_mask)
163 {
164 uint64_t val = 0U;
165
166 /* clean the previous response state */
167 ari_clobber_response(ari_base);
168
169 /* update CLUSTER_CSTATE? */
170 if (cluster != 0U) {
171 val |= (cluster & CLUSTER_CSTATE_MASK) |
172 CLUSTER_CSTATE_UPDATE_BIT;
173 }
174
175 /* update CCPLEX_CSTATE? */
176 if (ccplex != 0U) {
177 val |= ((ccplex & CCPLEX_CSTATE_MASK) << CCPLEX_CSTATE_SHIFT) |
178 CCPLEX_CSTATE_UPDATE_BIT;
179 }
180
181 /* update SYSTEM_CSTATE? */
182 if (system != 0U) {
183 val |= ((system & SYSTEM_CSTATE_MASK) << SYSTEM_CSTATE_SHIFT) |
184 (((uint64_t)sys_state_force << SYSTEM_CSTATE_FORCE_UPDATE_SHIFT) |
185 SYSTEM_CSTATE_UPDATE_BIT);
186 }
187
188 /* update wake mask value? */
189 if (update_wake_mask != 0U) {
190 val |= CSTATE_WAKE_MASK_UPDATE_BIT;
191 }
192
193 /* set the updated cstate info */
194 return ari_request_wait(ari_base, 0U, (uint32_t)TEGRA_ARI_UPDATE_CSTATE_INFO,
195 (uint32_t)val, wake_mask);
196 }
197
ari_update_crossover_time(uint32_t ari_base,uint32_t type,uint32_t time)198 int32_t ari_update_crossover_time(uint32_t ari_base, uint32_t type, uint32_t time)
199 {
200 int32_t ret = 0;
201
202 /* sanity check crossover type */
203 if ((type == TEGRA_ARI_CROSSOVER_C1_C6) ||
204 (type > TEGRA_ARI_CROSSOVER_CCP3_SC1)) {
205 ret = EINVAL;
206 } else {
207 /* clean the previous response state */
208 ari_clobber_response(ari_base);
209
210 /* update crossover threshold time */
211 ret = ari_request_wait(ari_base, 0U,
212 (uint32_t)TEGRA_ARI_UPDATE_CROSSOVER, type, time);
213 }
214
215 return ret;
216 }
217
ari_read_cstate_stats(uint32_t ari_base,uint32_t state)218 uint64_t ari_read_cstate_stats(uint32_t ari_base, uint32_t state)
219 {
220 int32_t ret;
221 uint64_t result;
222
223 /* sanity check crossover type */
224 if (state == 0U) {
225 result = EINVAL;
226 } else {
227 /* clean the previous response state */
228 ari_clobber_response(ari_base);
229
230 ret = ari_request_wait(ari_base, 0U,
231 (uint32_t)TEGRA_ARI_CSTATE_STATS, state, 0U);
232 if (ret != 0) {
233 result = EINVAL;
234 } else {
235 result = (uint64_t)ari_get_response_low(ari_base);
236 }
237 }
238 return result;
239 }
240
ari_write_cstate_stats(uint32_t ari_base,uint32_t state,uint32_t stats)241 int32_t ari_write_cstate_stats(uint32_t ari_base, uint32_t state, uint32_t stats)
242 {
243 /* clean the previous response state */
244 ari_clobber_response(ari_base);
245
246 /* write the cstate stats */
247 return ari_request_wait(ari_base, 0U, (uint32_t)TEGRA_ARI_WRITE_CSTATE_STATS,
248 state, stats);
249 }
250
ari_enumeration_misc(uint32_t ari_base,uint32_t cmd,uint32_t data)251 uint64_t ari_enumeration_misc(uint32_t ari_base, uint32_t cmd, uint32_t data)
252 {
253 uint64_t resp;
254 int32_t ret;
255 uint32_t local_data = data;
256
257 /* clean the previous response state */
258 ari_clobber_response(ari_base);
259
260 /* ARI_REQUEST_DATA_HI is reserved for commands other than 'ECHO' */
261 if (cmd != TEGRA_ARI_MISC_ECHO) {
262 local_data = 0U;
263 }
264
265 ret = ari_request_wait(ari_base, 0U, (uint32_t)TEGRA_ARI_MISC, cmd, local_data);
266 if (ret != 0) {
267 resp = (uint64_t)ret;
268 } else {
269 /* get the command response */
270 resp = ari_get_response_low(ari_base);
271 resp |= ((uint64_t)ari_get_response_high(ari_base) << 32);
272 }
273
274 return resp;
275 }
276
ari_is_ccx_allowed(uint32_t ari_base,uint32_t state,uint32_t wake_time)277 int32_t ari_is_ccx_allowed(uint32_t ari_base, uint32_t state, uint32_t wake_time)
278 {
279 int32_t ret;
280 uint32_t result;
281
282 /* clean the previous response state */
283 ari_clobber_response(ari_base);
284
285 ret = ari_request_wait(ari_base, 0U, (uint32_t)TEGRA_ARI_IS_CCX_ALLOWED,
286 state & 0x7U, wake_time);
287 if (ret != 0) {
288 ERROR("%s: failed (%d)\n", __func__, ret);
289 result = 0U;
290 } else {
291 result = ari_get_response_low(ari_base) & 0x1U;
292 }
293
294 /* 1 = CCx allowed, 0 = CCx not allowed */
295 return (int32_t)result;
296 }
297
ari_is_sc7_allowed(uint32_t ari_base,uint32_t state,uint32_t wake_time)298 int32_t ari_is_sc7_allowed(uint32_t ari_base, uint32_t state, uint32_t wake_time)
299 {
300 int32_t ret, result;
301
302 /* check for allowed power state */
303 if ((state != TEGRA_ARI_CORE_C0) && (state != TEGRA_ARI_CORE_C1) &&
304 (state != TEGRA_ARI_CORE_C6) && (state != TEGRA_ARI_CORE_C7)) {
305 ERROR("%s: unknown cstate (%d)\n", __func__, state);
306 result = EINVAL;
307 } else {
308 /* clean the previous response state */
309 ari_clobber_response(ari_base);
310
311 ret = ari_request_wait(ari_base, 0U,
312 (uint32_t)TEGRA_ARI_IS_SC7_ALLOWED, state, wake_time);
313 if (ret != 0) {
314 ERROR("%s: failed (%d)\n", __func__, ret);
315 result = 0;
316 } else {
317 /* 1 = SC7 allowed, 0 = SC7 not allowed */
318 result = (ari_get_response_low(ari_base) != 0U) ? 1 : 0;
319 }
320 }
321
322 return result;
323 }
324
ari_online_core(uint32_t ari_base,uint32_t core)325 int32_t ari_online_core(uint32_t ari_base, uint32_t core)
326 {
327 uint64_t cpu = read_mpidr() & (MPIDR_CPU_MASK);
328 uint64_t cluster = (read_mpidr() & (MPIDR_CLUSTER_MASK)) >>
329 (MPIDR_AFFINITY_BITS);
330 uint64_t impl = (read_midr() >> MIDR_IMPL_SHIFT) & MIDR_IMPL_MASK;
331 int32_t ret;
332
333 /* construct the current CPU # */
334 cpu |= (cluster << 2);
335
336 /* sanity check target core id */
337 if ((core >= MCE_CORE_ID_MAX) || (cpu == (uint64_t)core)) {
338 ERROR("%s: unsupported core id (%d)\n", __func__, core);
339 ret = EINVAL;
340 } else {
341 /*
342 * The Denver cluster has 2 CPUs only - 0, 1.
343 */
344 if ((impl == DENVER_IMPL) && ((core == 2U) || (core == 3U))) {
345 ERROR("%s: unknown core id (%d)\n", __func__, core);
346 ret = EINVAL;
347 } else {
348 /* clean the previous response state */
349 ari_clobber_response(ari_base);
350 ret = ari_request_wait(ari_base, 0U,
351 (uint32_t)TEGRA_ARI_ONLINE_CORE, core, 0U);
352 }
353 }
354
355 return ret;
356 }
357
ari_cc3_ctrl(uint32_t ari_base,uint32_t freq,uint32_t volt,uint8_t enable)358 int32_t ari_cc3_ctrl(uint32_t ari_base, uint32_t freq, uint32_t volt, uint8_t enable)
359 {
360 uint32_t val;
361
362 /* clean the previous response state */
363 ari_clobber_response(ari_base);
364
365 /*
366 * If the enable bit is cleared, Auto-CC3 will be disabled by setting
367 * the SW visible voltage/frequency request registers for all non
368 * floorswept cores valid independent of StandbyWFI and disabling
369 * the IDLE voltage/frequency request register. If set, Auto-CC3
370 * will be enabled by setting the ARM SW visible voltage/frequency
371 * request registers for all non floorswept cores to be enabled by
372 * StandbyWFI or the equivalent signal, and always keeping the IDLE
373 * voltage/frequency request register enabled.
374 */
375 val = (((freq & MCE_AUTO_CC3_FREQ_MASK) << MCE_AUTO_CC3_FREQ_SHIFT) |\
376 ((volt & MCE_AUTO_CC3_VTG_MASK) << MCE_AUTO_CC3_VTG_SHIFT) |\
377 ((enable != 0U) ? MCE_AUTO_CC3_ENABLE_BIT : 0U));
378
379 return ari_request_wait(ari_base, 0U,
380 (uint32_t)TEGRA_ARI_CC3_CTRL, val, 0U);
381 }
382
ari_reset_vector_update(uint32_t ari_base)383 int32_t ari_reset_vector_update(uint32_t ari_base)
384 {
385 /* clean the previous response state */
386 ari_clobber_response(ari_base);
387
388 /*
389 * Need to program the CPU reset vector one time during cold boot
390 * and SC7 exit
391 */
392 (void)ari_request_wait(ari_base, 0U,
393 (uint32_t)TEGRA_ARI_COPY_MISCREG_AA64_RST, 0U, 0U);
394
395 return 0;
396 }
397
ari_roc_flush_cache_trbits(uint32_t ari_base)398 int32_t ari_roc_flush_cache_trbits(uint32_t ari_base)
399 {
400 /* clean the previous response state */
401 ari_clobber_response(ari_base);
402
403 return ari_request_wait(ari_base, 0U,
404 (uint32_t)TEGRA_ARI_ROC_FLUSH_CACHE_TRBITS, 0U, 0U);
405 }
406
ari_roc_flush_cache(uint32_t ari_base)407 int32_t ari_roc_flush_cache(uint32_t ari_base)
408 {
409 /* clean the previous response state */
410 ari_clobber_response(ari_base);
411
412 return ari_request_wait(ari_base, 0U,
413 (uint32_t)TEGRA_ARI_ROC_FLUSH_CACHE_ONLY, 0U, 0U);
414 }
415
ari_roc_clean_cache(uint32_t ari_base)416 int32_t ari_roc_clean_cache(uint32_t ari_base)
417 {
418 /* clean the previous response state */
419 ari_clobber_response(ari_base);
420
421 return ari_request_wait(ari_base, 0U,
422 (uint32_t)TEGRA_ARI_ROC_CLEAN_CACHE_ONLY, 0U, 0U);
423 }
424
ari_read_write_mca(uint32_t ari_base,uint64_t cmd,uint64_t * data)425 uint64_t ari_read_write_mca(uint32_t ari_base, uint64_t cmd, uint64_t *data)
426 {
427 uint64_t mca_arg_data, result = 0;
428 uint32_t resp_lo, resp_hi;
429 uint32_t mca_arg_err, mca_arg_finish;
430 int32_t ret;
431
432 /* Set data (write) */
433 mca_arg_data = (data != NULL) ? *data : 0ULL;
434
435 /* Set command */
436 ari_write_32(ari_base, (uint32_t)cmd, ARI_RESPONSE_DATA_LO);
437 ari_write_32(ari_base, (uint32_t)(cmd >> 32U), ARI_RESPONSE_DATA_HI);
438
439 ret = ari_request_wait(ari_base, 0U, (uint32_t)TEGRA_ARI_MCA,
440 (uint32_t)mca_arg_data,
441 (uint32_t)(mca_arg_data >> 32U));
442 if (ret == 0) {
443 resp_lo = ari_get_response_low(ari_base);
444 resp_hi = ari_get_response_high(ari_base);
445
446 mca_arg_err = resp_lo & MCA_ARG_ERROR_MASK;
447 mca_arg_finish = (resp_hi >> MCA_ARG_FINISH_SHIFT) &
448 MCA_ARG_FINISH_MASK;
449
450 if (mca_arg_finish == 0U) {
451 result = (uint64_t)mca_arg_err;
452 } else {
453 if (data != NULL) {
454 resp_lo = ari_get_request_low(ari_base);
455 resp_hi = ari_get_request_high(ari_base);
456 *data = ((uint64_t)resp_hi << 32U) |
457 (uint64_t)resp_lo;
458 }
459 }
460 }
461
462 return result;
463 }
464
ari_update_ccplex_gsc(uint32_t ari_base,uint32_t gsc_idx)465 int32_t ari_update_ccplex_gsc(uint32_t ari_base, uint32_t gsc_idx)
466 {
467 int32_t ret = 0;
468 /* sanity check GSC ID */
469 if (gsc_idx > TEGRA_ARI_GSC_VPR_IDX) {
470 ret = EINVAL;
471 } else {
472 /* clean the previous response state */
473 ari_clobber_response(ari_base);
474
475 /*
476 * The MCE code will read the GSC carveout value, corrseponding to
477 * the ID, from the MC registers and update the internal GSC registers
478 * of the CCPLEX.
479 */
480 (void)ari_request_wait(ari_base, 0U,
481 (uint32_t)TEGRA_ARI_UPDATE_CCPLEX_GSC, gsc_idx, 0U);
482 }
483
484 return ret;
485 }
486
ari_enter_ccplex_state(uint32_t ari_base,uint32_t state_idx)487 void ari_enter_ccplex_state(uint32_t ari_base, uint32_t state_idx)
488 {
489 /* clean the previous response state */
490 ari_clobber_response(ari_base);
491
492 /*
493 * The MCE will shutdown or restart the entire system
494 */
495 (void)ari_request_wait(ari_base, 0U,
496 (uint32_t)TEGRA_ARI_MISC_CCPLEX, state_idx, 0U);
497 }
498
ari_read_write_uncore_perfmon(uint32_t ari_base,uint64_t req,uint64_t * data)499 int32_t ari_read_write_uncore_perfmon(uint32_t ari_base, uint64_t req,
500 uint64_t *data)
501 {
502 int32_t ret, result;
503 uint32_t val, req_status;
504 uint8_t req_cmd;
505
506 req_cmd = (uint8_t)(req & UNCORE_PERFMON_CMD_MASK);
507
508 /* clean the previous response state */
509 ari_clobber_response(ari_base);
510
511 /* sanity check input parameters */
512 if ((req_cmd == UNCORE_PERFMON_CMD_READ) && (data == NULL)) {
513 ERROR("invalid parameters\n");
514 result = EINVAL;
515 } else {
516 /*
517 * For "write" commands get the value that has to be written
518 * to the uncore perfmon registers
519 */
520 val = (req_cmd == UNCORE_PERFMON_CMD_WRITE) ?
521 (uint32_t)*data : 0U;
522
523 ret = ari_request_wait(ari_base, 0U,
524 (uint32_t)TEGRA_ARI_PERFMON, val, (uint32_t)req);
525 if (ret != 0) {
526 result = ret;
527 } else {
528 /* read the command status value */
529 req_status = ari_get_response_high(ari_base) &
530 UNCORE_PERFMON_RESP_STATUS_MASK;
531
532 /*
533 * For "read" commands get the data from the uncore
534 * perfmon registers
535 */
536 req_status &= UNCORE_PERFMON_RESP_STATUS_MASK;
537 if ((req_status == 0U) && (req_cmd == UNCORE_PERFMON_CMD_READ)) {
538 *data = ari_get_response_low(ari_base);
539 }
540 result = (int32_t)req_status;
541 }
542 }
543
544 return result;
545 }
546
ari_misc_ccplex(uint32_t ari_base,uint32_t index,uint32_t value)547 void ari_misc_ccplex(uint32_t ari_base, uint32_t index, uint32_t value)
548 {
549 /*
550 * This invokes the ARI_MISC_CCPLEX commands. This can be
551 * used to enable/disable coresight clock gating.
552 */
553
554 if ((index > TEGRA_ARI_MISC_CCPLEX_EDBGREQ) ||
555 ((index == TEGRA_ARI_MISC_CCPLEX_CORESIGHT_CG_CTRL) &&
556 (value > 1U))) {
557 ERROR("%s: invalid parameters \n", __func__);
558 } else {
559 /* clean the previous response state */
560 ari_clobber_response(ari_base);
561 (void)ari_request_wait(ari_base, 0U,
562 (uint32_t)TEGRA_ARI_MISC_CCPLEX, index, value);
563 }
564 }
565