1 /*
2 * Copyright(c) 2015 - 2017 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47
48 #include <linux/firmware.h>
49 #include <linux/mutex.h>
50 #include <linux/module.h>
51 #include <linux/delay.h>
52 #include <linux/crc32.h>
53
54 #include "hfi.h"
55 #include "trace.h"
56
57 /*
58 * Make it easy to toggle firmware file name and if it gets loaded by
59 * editing the following. This may be something we do while in development
60 * but not necessarily something a user would ever need to use.
61 */
62 #define DEFAULT_FW_8051_NAME_FPGA "hfi_dc8051.bin"
63 #define DEFAULT_FW_8051_NAME_ASIC "hfi1_dc8051.fw"
64 #define DEFAULT_FW_FABRIC_NAME "hfi1_fabric.fw"
65 #define DEFAULT_FW_SBUS_NAME "hfi1_sbus.fw"
66 #define DEFAULT_FW_PCIE_NAME "hfi1_pcie.fw"
67 #define ALT_FW_8051_NAME_ASIC "hfi1_dc8051_d.fw"
68 #define ALT_FW_FABRIC_NAME "hfi1_fabric_d.fw"
69 #define ALT_FW_SBUS_NAME "hfi1_sbus_d.fw"
70 #define ALT_FW_PCIE_NAME "hfi1_pcie_d.fw"
71
72 MODULE_FIRMWARE(DEFAULT_FW_8051_NAME_ASIC);
73 MODULE_FIRMWARE(DEFAULT_FW_FABRIC_NAME);
74 MODULE_FIRMWARE(DEFAULT_FW_SBUS_NAME);
75 MODULE_FIRMWARE(DEFAULT_FW_PCIE_NAME);
76
77 static uint fw_8051_load = 1;
78 static uint fw_fabric_serdes_load = 1;
79 static uint fw_pcie_serdes_load = 1;
80 static uint fw_sbus_load = 1;
81
82 /* Firmware file names get set in hfi1_firmware_init() based on the above */
83 static char *fw_8051_name;
84 static char *fw_fabric_serdes_name;
85 static char *fw_sbus_name;
86 static char *fw_pcie_serdes_name;
87
88 #define SBUS_MAX_POLL_COUNT 100
89 #define SBUS_COUNTER(reg, name) \
90 (((reg) >> ASIC_STS_SBUS_COUNTERS_##name##_CNT_SHIFT) & \
91 ASIC_STS_SBUS_COUNTERS_##name##_CNT_MASK)
92
93 /*
94 * Firmware security header.
95 */
96 struct css_header {
97 u32 module_type;
98 u32 header_len;
99 u32 header_version;
100 u32 module_id;
101 u32 module_vendor;
102 u32 date; /* BCD yyyymmdd */
103 u32 size; /* in DWORDs */
104 u32 key_size; /* in DWORDs */
105 u32 modulus_size; /* in DWORDs */
106 u32 exponent_size; /* in DWORDs */
107 u32 reserved[22];
108 };
109
110 /* expected field values */
111 #define CSS_MODULE_TYPE 0x00000006
112 #define CSS_HEADER_LEN 0x000000a1
113 #define CSS_HEADER_VERSION 0x00010000
114 #define CSS_MODULE_VENDOR 0x00008086
115
116 #define KEY_SIZE 256
117 #define MU_SIZE 8
118 #define EXPONENT_SIZE 4
119
120 /* size of platform configuration partition */
121 #define MAX_PLATFORM_CONFIG_FILE_SIZE 4096
122
123 /* size of file of plaform configuration encoded in format version 4 */
124 #define PLATFORM_CONFIG_FORMAT_4_FILE_SIZE 528
125
126 /* the file itself */
127 struct firmware_file {
128 struct css_header css_header;
129 u8 modulus[KEY_SIZE];
130 u8 exponent[EXPONENT_SIZE];
131 u8 signature[KEY_SIZE];
132 u8 firmware[];
133 };
134
135 struct augmented_firmware_file {
136 struct css_header css_header;
137 u8 modulus[KEY_SIZE];
138 u8 exponent[EXPONENT_SIZE];
139 u8 signature[KEY_SIZE];
140 u8 r2[KEY_SIZE];
141 u8 mu[MU_SIZE];
142 u8 firmware[];
143 };
144
145 /* augmented file size difference */
146 #define AUGMENT_SIZE (sizeof(struct augmented_firmware_file) - \
147 sizeof(struct firmware_file))
148
149 struct firmware_details {
150 /* Linux core piece */
151 const struct firmware *fw;
152
153 struct css_header *css_header;
154 u8 *firmware_ptr; /* pointer to binary data */
155 u32 firmware_len; /* length in bytes */
156 u8 *modulus; /* pointer to the modulus */
157 u8 *exponent; /* pointer to the exponent */
158 u8 *signature; /* pointer to the signature */
159 u8 *r2; /* pointer to r2 */
160 u8 *mu; /* pointer to mu */
161 struct augmented_firmware_file dummy_header;
162 };
163
164 /*
165 * The mutex protects fw_state, fw_err, and all of the firmware_details
166 * variables.
167 */
168 static DEFINE_MUTEX(fw_mutex);
169 enum fw_state {
170 FW_EMPTY,
171 FW_TRY,
172 FW_FINAL,
173 FW_ERR
174 };
175
176 static enum fw_state fw_state = FW_EMPTY;
177 static int fw_err;
178 static struct firmware_details fw_8051;
179 static struct firmware_details fw_fabric;
180 static struct firmware_details fw_pcie;
181 static struct firmware_details fw_sbus;
182
183 /* flags for turn_off_spicos() */
184 #define SPICO_SBUS 0x1
185 #define SPICO_FABRIC 0x2
186 #define ENABLE_SPICO_SMASK 0x1
187
188 /* security block commands */
189 #define RSA_CMD_INIT 0x1
190 #define RSA_CMD_START 0x2
191
192 /* security block status */
193 #define RSA_STATUS_IDLE 0x0
194 #define RSA_STATUS_ACTIVE 0x1
195 #define RSA_STATUS_DONE 0x2
196 #define RSA_STATUS_FAILED 0x3
197
198 /* RSA engine timeout, in ms */
199 #define RSA_ENGINE_TIMEOUT 100 /* ms */
200
201 /* hardware mutex timeout, in ms */
202 #define HM_TIMEOUT 10 /* ms */
203
204 /* 8051 memory access timeout, in us */
205 #define DC8051_ACCESS_TIMEOUT 100 /* us */
206
207 /* the number of fabric SerDes on the SBus */
208 #define NUM_FABRIC_SERDES 4
209
210 /* ASIC_STS_SBUS_RESULT.RESULT_CODE value */
211 #define SBUS_READ_COMPLETE 0x4
212
213 /* SBus fabric SerDes addresses, one set per HFI */
214 static const u8 fabric_serdes_addrs[2][NUM_FABRIC_SERDES] = {
215 { 0x01, 0x02, 0x03, 0x04 },
216 { 0x28, 0x29, 0x2a, 0x2b }
217 };
218
219 /* SBus PCIe SerDes addresses, one set per HFI */
220 static const u8 pcie_serdes_addrs[2][NUM_PCIE_SERDES] = {
221 { 0x08, 0x0a, 0x0c, 0x0e, 0x10, 0x12, 0x14, 0x16,
222 0x18, 0x1a, 0x1c, 0x1e, 0x20, 0x22, 0x24, 0x26 },
223 { 0x2f, 0x31, 0x33, 0x35, 0x37, 0x39, 0x3b, 0x3d,
224 0x3f, 0x41, 0x43, 0x45, 0x47, 0x49, 0x4b, 0x4d }
225 };
226
227 /* SBus PCIe PCS addresses, one set per HFI */
228 const u8 pcie_pcs_addrs[2][NUM_PCIE_SERDES] = {
229 { 0x09, 0x0b, 0x0d, 0x0f, 0x11, 0x13, 0x15, 0x17,
230 0x19, 0x1b, 0x1d, 0x1f, 0x21, 0x23, 0x25, 0x27 },
231 { 0x30, 0x32, 0x34, 0x36, 0x38, 0x3a, 0x3c, 0x3e,
232 0x40, 0x42, 0x44, 0x46, 0x48, 0x4a, 0x4c, 0x4e }
233 };
234
235 /* SBus fabric SerDes broadcast addresses, one per HFI */
236 static const u8 fabric_serdes_broadcast[2] = { 0xe4, 0xe5 };
237 static const u8 all_fabric_serdes_broadcast = 0xe1;
238
239 /* SBus PCIe SerDes broadcast addresses, one per HFI */
240 const u8 pcie_serdes_broadcast[2] = { 0xe2, 0xe3 };
241 static const u8 all_pcie_serdes_broadcast = 0xe0;
242
243 static const u32 platform_config_table_limits[PLATFORM_CONFIG_TABLE_MAX] = {
244 0,
245 SYSTEM_TABLE_MAX,
246 PORT_TABLE_MAX,
247 RX_PRESET_TABLE_MAX,
248 TX_PRESET_TABLE_MAX,
249 QSFP_ATTEN_TABLE_MAX,
250 VARIABLE_SETTINGS_TABLE_MAX
251 };
252
253 /* forwards */
254 static void dispose_one_firmware(struct firmware_details *fdet);
255 static int load_fabric_serdes_firmware(struct hfi1_devdata *dd,
256 struct firmware_details *fdet);
257 static void dump_fw_version(struct hfi1_devdata *dd);
258
259 /*
260 * Read a single 64-bit value from 8051 data memory.
261 *
262 * Expects:
263 * o caller to have already set up data read, no auto increment
264 * o caller to turn off read enable when finished
265 *
266 * The address argument is a byte offset. Bits 0:2 in the address are
267 * ignored - i.e. the hardware will always do aligned 8-byte reads as if
268 * the lower bits are zero.
269 *
270 * Return 0 on success, -ENXIO on a read error (timeout).
271 */
__read_8051_data(struct hfi1_devdata * dd,u32 addr,u64 * result)272 static int __read_8051_data(struct hfi1_devdata *dd, u32 addr, u64 *result)
273 {
274 u64 reg;
275 int count;
276
277 /* step 1: set the address, clear enable */
278 reg = (addr & DC_DC8051_CFG_RAM_ACCESS_CTRL_ADDRESS_MASK)
279 << DC_DC8051_CFG_RAM_ACCESS_CTRL_ADDRESS_SHIFT;
280 write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_CTRL, reg);
281 /* step 2: enable */
282 write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_CTRL,
283 reg | DC_DC8051_CFG_RAM_ACCESS_CTRL_READ_ENA_SMASK);
284
285 /* wait until ACCESS_COMPLETED is set */
286 count = 0;
287 while ((read_csr(dd, DC_DC8051_CFG_RAM_ACCESS_STATUS)
288 & DC_DC8051_CFG_RAM_ACCESS_STATUS_ACCESS_COMPLETED_SMASK)
289 == 0) {
290 count++;
291 if (count > DC8051_ACCESS_TIMEOUT) {
292 dd_dev_err(dd, "timeout reading 8051 data\n");
293 return -ENXIO;
294 }
295 ndelay(10);
296 }
297
298 /* gather the data */
299 *result = read_csr(dd, DC_DC8051_CFG_RAM_ACCESS_RD_DATA);
300
301 return 0;
302 }
303
304 /*
305 * Read 8051 data starting at addr, for len bytes. Will read in 8-byte chunks.
306 * Return 0 on success, -errno on error.
307 */
read_8051_data(struct hfi1_devdata * dd,u32 addr,u32 len,u64 * result)308 int read_8051_data(struct hfi1_devdata *dd, u32 addr, u32 len, u64 *result)
309 {
310 unsigned long flags;
311 u32 done;
312 int ret = 0;
313
314 spin_lock_irqsave(&dd->dc8051_memlock, flags);
315
316 /* data read set-up, no auto-increment */
317 write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_SETUP, 0);
318
319 for (done = 0; done < len; addr += 8, done += 8, result++) {
320 ret = __read_8051_data(dd, addr, result);
321 if (ret)
322 break;
323 }
324
325 /* turn off read enable */
326 write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_CTRL, 0);
327
328 spin_unlock_irqrestore(&dd->dc8051_memlock, flags);
329
330 return ret;
331 }
332
333 /*
334 * Write data or code to the 8051 code or data RAM.
335 */
write_8051(struct hfi1_devdata * dd,int code,u32 start,const u8 * data,u32 len)336 static int write_8051(struct hfi1_devdata *dd, int code, u32 start,
337 const u8 *data, u32 len)
338 {
339 u64 reg;
340 u32 offset;
341 int aligned, count;
342
343 /* check alignment */
344 aligned = ((unsigned long)data & 0x7) == 0;
345
346 /* write set-up */
347 reg = (code ? DC_DC8051_CFG_RAM_ACCESS_SETUP_RAM_SEL_SMASK : 0ull)
348 | DC_DC8051_CFG_RAM_ACCESS_SETUP_AUTO_INCR_ADDR_SMASK;
349 write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_SETUP, reg);
350
351 reg = ((start & DC_DC8051_CFG_RAM_ACCESS_CTRL_ADDRESS_MASK)
352 << DC_DC8051_CFG_RAM_ACCESS_CTRL_ADDRESS_SHIFT)
353 | DC_DC8051_CFG_RAM_ACCESS_CTRL_WRITE_ENA_SMASK;
354 write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_CTRL, reg);
355
356 /* write */
357 for (offset = 0; offset < len; offset += 8) {
358 int bytes = len - offset;
359
360 if (bytes < 8) {
361 reg = 0;
362 memcpy(®, &data[offset], bytes);
363 } else if (aligned) {
364 reg = *(u64 *)&data[offset];
365 } else {
366 memcpy(®, &data[offset], 8);
367 }
368 write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_WR_DATA, reg);
369
370 /* wait until ACCESS_COMPLETED is set */
371 count = 0;
372 while ((read_csr(dd, DC_DC8051_CFG_RAM_ACCESS_STATUS)
373 & DC_DC8051_CFG_RAM_ACCESS_STATUS_ACCESS_COMPLETED_SMASK)
374 == 0) {
375 count++;
376 if (count > DC8051_ACCESS_TIMEOUT) {
377 dd_dev_err(dd, "timeout writing 8051 data\n");
378 return -ENXIO;
379 }
380 udelay(1);
381 }
382 }
383
384 /* turn off write access, auto increment (also sets to data access) */
385 write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_CTRL, 0);
386 write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_SETUP, 0);
387
388 return 0;
389 }
390
391 /* return 0 if values match, non-zero and complain otherwise */
invalid_header(struct hfi1_devdata * dd,const char * what,u32 actual,u32 expected)392 static int invalid_header(struct hfi1_devdata *dd, const char *what,
393 u32 actual, u32 expected)
394 {
395 if (actual == expected)
396 return 0;
397
398 dd_dev_err(dd,
399 "invalid firmware header field %s: expected 0x%x, actual 0x%x\n",
400 what, expected, actual);
401 return 1;
402 }
403
404 /*
405 * Verify that the static fields in the CSS header match.
406 */
verify_css_header(struct hfi1_devdata * dd,struct css_header * css)407 static int verify_css_header(struct hfi1_devdata *dd, struct css_header *css)
408 {
409 /* verify CSS header fields (most sizes are in DW, so add /4) */
410 if (invalid_header(dd, "module_type", css->module_type,
411 CSS_MODULE_TYPE) ||
412 invalid_header(dd, "header_len", css->header_len,
413 (sizeof(struct firmware_file) / 4)) ||
414 invalid_header(dd, "header_version", css->header_version,
415 CSS_HEADER_VERSION) ||
416 invalid_header(dd, "module_vendor", css->module_vendor,
417 CSS_MODULE_VENDOR) ||
418 invalid_header(dd, "key_size", css->key_size, KEY_SIZE / 4) ||
419 invalid_header(dd, "modulus_size", css->modulus_size,
420 KEY_SIZE / 4) ||
421 invalid_header(dd, "exponent_size", css->exponent_size,
422 EXPONENT_SIZE / 4)) {
423 return -EINVAL;
424 }
425 return 0;
426 }
427
428 /*
429 * Make sure there are at least some bytes after the prefix.
430 */
payload_check(struct hfi1_devdata * dd,const char * name,long file_size,long prefix_size)431 static int payload_check(struct hfi1_devdata *dd, const char *name,
432 long file_size, long prefix_size)
433 {
434 /* make sure we have some payload */
435 if (prefix_size >= file_size) {
436 dd_dev_err(dd,
437 "firmware \"%s\", size %ld, must be larger than %ld bytes\n",
438 name, file_size, prefix_size);
439 return -EINVAL;
440 }
441
442 return 0;
443 }
444
445 /*
446 * Request the firmware from the system. Extract the pieces and fill in
447 * fdet. If successful, the caller will need to call dispose_one_firmware().
448 * Returns 0 on success, -ERRNO on error.
449 */
obtain_one_firmware(struct hfi1_devdata * dd,const char * name,struct firmware_details * fdet)450 static int obtain_one_firmware(struct hfi1_devdata *dd, const char *name,
451 struct firmware_details *fdet)
452 {
453 struct css_header *css;
454 int ret;
455
456 memset(fdet, 0, sizeof(*fdet));
457
458 ret = request_firmware(&fdet->fw, name, &dd->pcidev->dev);
459 if (ret) {
460 dd_dev_warn(dd, "cannot find firmware \"%s\", err %d\n",
461 name, ret);
462 return ret;
463 }
464
465 /* verify the firmware */
466 if (fdet->fw->size < sizeof(struct css_header)) {
467 dd_dev_err(dd, "firmware \"%s\" is too small\n", name);
468 ret = -EINVAL;
469 goto done;
470 }
471 css = (struct css_header *)fdet->fw->data;
472
473 hfi1_cdbg(FIRMWARE, "Firmware %s details:", name);
474 hfi1_cdbg(FIRMWARE, "file size: 0x%lx bytes", fdet->fw->size);
475 hfi1_cdbg(FIRMWARE, "CSS structure:");
476 hfi1_cdbg(FIRMWARE, " module_type 0x%x", css->module_type);
477 hfi1_cdbg(FIRMWARE, " header_len 0x%03x (0x%03x bytes)",
478 css->header_len, 4 * css->header_len);
479 hfi1_cdbg(FIRMWARE, " header_version 0x%x", css->header_version);
480 hfi1_cdbg(FIRMWARE, " module_id 0x%x", css->module_id);
481 hfi1_cdbg(FIRMWARE, " module_vendor 0x%x", css->module_vendor);
482 hfi1_cdbg(FIRMWARE, " date 0x%x", css->date);
483 hfi1_cdbg(FIRMWARE, " size 0x%03x (0x%03x bytes)",
484 css->size, 4 * css->size);
485 hfi1_cdbg(FIRMWARE, " key_size 0x%03x (0x%03x bytes)",
486 css->key_size, 4 * css->key_size);
487 hfi1_cdbg(FIRMWARE, " modulus_size 0x%03x (0x%03x bytes)",
488 css->modulus_size, 4 * css->modulus_size);
489 hfi1_cdbg(FIRMWARE, " exponent_size 0x%03x (0x%03x bytes)",
490 css->exponent_size, 4 * css->exponent_size);
491 hfi1_cdbg(FIRMWARE, "firmware size: 0x%lx bytes",
492 fdet->fw->size - sizeof(struct firmware_file));
493
494 /*
495 * If the file does not have a valid CSS header, fail.
496 * Otherwise, check the CSS size field for an expected size.
497 * The augmented file has r2 and mu inserted after the header
498 * was generated, so there will be a known difference between
499 * the CSS header size and the actual file size. Use this
500 * difference to identify an augmented file.
501 *
502 * Note: css->size is in DWORDs, multiply by 4 to get bytes.
503 */
504 ret = verify_css_header(dd, css);
505 if (ret) {
506 dd_dev_info(dd, "Invalid CSS header for \"%s\"\n", name);
507 } else if ((css->size * 4) == fdet->fw->size) {
508 /* non-augmented firmware file */
509 struct firmware_file *ff = (struct firmware_file *)
510 fdet->fw->data;
511
512 /* make sure there are bytes in the payload */
513 ret = payload_check(dd, name, fdet->fw->size,
514 sizeof(struct firmware_file));
515 if (ret == 0) {
516 fdet->css_header = css;
517 fdet->modulus = ff->modulus;
518 fdet->exponent = ff->exponent;
519 fdet->signature = ff->signature;
520 fdet->r2 = fdet->dummy_header.r2; /* use dummy space */
521 fdet->mu = fdet->dummy_header.mu; /* use dummy space */
522 fdet->firmware_ptr = ff->firmware;
523 fdet->firmware_len = fdet->fw->size -
524 sizeof(struct firmware_file);
525 /*
526 * Header does not include r2 and mu - generate here.
527 * For now, fail.
528 */
529 dd_dev_err(dd, "driver is unable to validate firmware without r2 and mu (not in firmware file)\n");
530 ret = -EINVAL;
531 }
532 } else if ((css->size * 4) + AUGMENT_SIZE == fdet->fw->size) {
533 /* augmented firmware file */
534 struct augmented_firmware_file *aff =
535 (struct augmented_firmware_file *)fdet->fw->data;
536
537 /* make sure there are bytes in the payload */
538 ret = payload_check(dd, name, fdet->fw->size,
539 sizeof(struct augmented_firmware_file));
540 if (ret == 0) {
541 fdet->css_header = css;
542 fdet->modulus = aff->modulus;
543 fdet->exponent = aff->exponent;
544 fdet->signature = aff->signature;
545 fdet->r2 = aff->r2;
546 fdet->mu = aff->mu;
547 fdet->firmware_ptr = aff->firmware;
548 fdet->firmware_len = fdet->fw->size -
549 sizeof(struct augmented_firmware_file);
550 }
551 } else {
552 /* css->size check failed */
553 dd_dev_err(dd,
554 "invalid firmware header field size: expected 0x%lx or 0x%lx, actual 0x%x\n",
555 fdet->fw->size / 4,
556 (fdet->fw->size - AUGMENT_SIZE) / 4,
557 css->size);
558
559 ret = -EINVAL;
560 }
561
562 done:
563 /* if returning an error, clean up after ourselves */
564 if (ret)
565 dispose_one_firmware(fdet);
566 return ret;
567 }
568
dispose_one_firmware(struct firmware_details * fdet)569 static void dispose_one_firmware(struct firmware_details *fdet)
570 {
571 release_firmware(fdet->fw);
572 /* erase all previous information */
573 memset(fdet, 0, sizeof(*fdet));
574 }
575
576 /*
577 * Obtain the 4 firmwares from the OS. All must be obtained at once or not
578 * at all. If called with the firmware state in FW_TRY, use alternate names.
579 * On exit, this routine will have set the firmware state to one of FW_TRY,
580 * FW_FINAL, or FW_ERR.
581 *
582 * Must be holding fw_mutex.
583 */
__obtain_firmware(struct hfi1_devdata * dd)584 static void __obtain_firmware(struct hfi1_devdata *dd)
585 {
586 int err = 0;
587
588 if (fw_state == FW_FINAL) /* nothing more to obtain */
589 return;
590 if (fw_state == FW_ERR) /* already in error */
591 return;
592
593 /* fw_state is FW_EMPTY or FW_TRY */
594 retry:
595 if (fw_state == FW_TRY) {
596 /*
597 * We tried the original and it failed. Move to the
598 * alternate.
599 */
600 dd_dev_warn(dd, "using alternate firmware names\n");
601 /*
602 * Let others run. Some systems, when missing firmware, does
603 * something that holds for 30 seconds. If we do that twice
604 * in a row it triggers task blocked warning.
605 */
606 cond_resched();
607 if (fw_8051_load)
608 dispose_one_firmware(&fw_8051);
609 if (fw_fabric_serdes_load)
610 dispose_one_firmware(&fw_fabric);
611 if (fw_sbus_load)
612 dispose_one_firmware(&fw_sbus);
613 if (fw_pcie_serdes_load)
614 dispose_one_firmware(&fw_pcie);
615 fw_8051_name = ALT_FW_8051_NAME_ASIC;
616 fw_fabric_serdes_name = ALT_FW_FABRIC_NAME;
617 fw_sbus_name = ALT_FW_SBUS_NAME;
618 fw_pcie_serdes_name = ALT_FW_PCIE_NAME;
619
620 /*
621 * Add a delay before obtaining and loading debug firmware.
622 * Authorization will fail if the delay between firmware
623 * authorization events is shorter than 50us. Add 100us to
624 * make a delay time safe.
625 */
626 usleep_range(100, 120);
627 }
628
629 if (fw_sbus_load) {
630 err = obtain_one_firmware(dd, fw_sbus_name, &fw_sbus);
631 if (err)
632 goto done;
633 }
634
635 if (fw_pcie_serdes_load) {
636 err = obtain_one_firmware(dd, fw_pcie_serdes_name, &fw_pcie);
637 if (err)
638 goto done;
639 }
640
641 if (fw_fabric_serdes_load) {
642 err = obtain_one_firmware(dd, fw_fabric_serdes_name,
643 &fw_fabric);
644 if (err)
645 goto done;
646 }
647
648 if (fw_8051_load) {
649 err = obtain_one_firmware(dd, fw_8051_name, &fw_8051);
650 if (err)
651 goto done;
652 }
653
654 done:
655 if (err) {
656 /* oops, had problems obtaining a firmware */
657 if (fw_state == FW_EMPTY && dd->icode == ICODE_RTL_SILICON) {
658 /* retry with alternate (RTL only) */
659 fw_state = FW_TRY;
660 goto retry;
661 }
662 dd_dev_err(dd, "unable to obtain working firmware\n");
663 fw_state = FW_ERR;
664 fw_err = -ENOENT;
665 } else {
666 /* success */
667 if (fw_state == FW_EMPTY &&
668 dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
669 fw_state = FW_TRY; /* may retry later */
670 else
671 fw_state = FW_FINAL; /* cannot try again */
672 }
673 }
674
675 /*
676 * Called by all HFIs when loading their firmware - i.e. device probe time.
677 * The first one will do the actual firmware load. Use a mutex to resolve
678 * any possible race condition.
679 *
680 * The call to this routine cannot be moved to driver load because the kernel
681 * call request_firmware() requires a device which is only available after
682 * the first device probe.
683 */
obtain_firmware(struct hfi1_devdata * dd)684 static int obtain_firmware(struct hfi1_devdata *dd)
685 {
686 unsigned long timeout;
687
688 mutex_lock(&fw_mutex);
689
690 /* 40s delay due to long delay on missing firmware on some systems */
691 timeout = jiffies + msecs_to_jiffies(40000);
692 while (fw_state == FW_TRY) {
693 /*
694 * Another device is trying the firmware. Wait until it
695 * decides what works (or not).
696 */
697 if (time_after(jiffies, timeout)) {
698 /* waited too long */
699 dd_dev_err(dd, "Timeout waiting for firmware try");
700 fw_state = FW_ERR;
701 fw_err = -ETIMEDOUT;
702 break;
703 }
704 mutex_unlock(&fw_mutex);
705 msleep(20); /* arbitrary delay */
706 mutex_lock(&fw_mutex);
707 }
708 /* not in FW_TRY state */
709
710 /* set fw_state to FW_TRY, FW_FINAL, or FW_ERR, and fw_err */
711 if (fw_state == FW_EMPTY)
712 __obtain_firmware(dd);
713
714 mutex_unlock(&fw_mutex);
715 return fw_err;
716 }
717
718 /*
719 * Called when the driver unloads. The timing is asymmetric with its
720 * counterpart, obtain_firmware(). If called at device remove time,
721 * then it is conceivable that another device could probe while the
722 * firmware is being disposed. The mutexes can be moved to do that
723 * safely, but then the firmware would be requested from the OS multiple
724 * times.
725 *
726 * No mutex is needed as the driver is unloading and there cannot be any
727 * other callers.
728 */
dispose_firmware(void)729 void dispose_firmware(void)
730 {
731 dispose_one_firmware(&fw_8051);
732 dispose_one_firmware(&fw_fabric);
733 dispose_one_firmware(&fw_pcie);
734 dispose_one_firmware(&fw_sbus);
735
736 /* retain the error state, otherwise revert to empty */
737 if (fw_state != FW_ERR)
738 fw_state = FW_EMPTY;
739 }
740
741 /*
742 * Called with the result of a firmware download.
743 *
744 * Return 1 to retry loading the firmware, 0 to stop.
745 */
retry_firmware(struct hfi1_devdata * dd,int load_result)746 static int retry_firmware(struct hfi1_devdata *dd, int load_result)
747 {
748 int retry;
749
750 mutex_lock(&fw_mutex);
751
752 if (load_result == 0) {
753 /*
754 * The load succeeded, so expect all others to do the same.
755 * Do not retry again.
756 */
757 if (fw_state == FW_TRY)
758 fw_state = FW_FINAL;
759 retry = 0; /* do NOT retry */
760 } else if (fw_state == FW_TRY) {
761 /* load failed, obtain alternate firmware */
762 __obtain_firmware(dd);
763 retry = (fw_state == FW_FINAL);
764 } else {
765 /* else in FW_FINAL or FW_ERR, no retry in either case */
766 retry = 0;
767 }
768
769 mutex_unlock(&fw_mutex);
770 return retry;
771 }
772
773 /*
774 * Write a block of data to a given array CSR. All calls will be in
775 * multiples of 8 bytes.
776 */
write_rsa_data(struct hfi1_devdata * dd,int what,const u8 * data,int nbytes)777 static void write_rsa_data(struct hfi1_devdata *dd, int what,
778 const u8 *data, int nbytes)
779 {
780 int qw_size = nbytes / 8;
781 int i;
782
783 if (((unsigned long)data & 0x7) == 0) {
784 /* aligned */
785 u64 *ptr = (u64 *)data;
786
787 for (i = 0; i < qw_size; i++, ptr++)
788 write_csr(dd, what + (8 * i), *ptr);
789 } else {
790 /* not aligned */
791 for (i = 0; i < qw_size; i++, data += 8) {
792 u64 value;
793
794 memcpy(&value, data, 8);
795 write_csr(dd, what + (8 * i), value);
796 }
797 }
798 }
799
800 /*
801 * Write a block of data to a given CSR as a stream of writes. All calls will
802 * be in multiples of 8 bytes.
803 */
write_streamed_rsa_data(struct hfi1_devdata * dd,int what,const u8 * data,int nbytes)804 static void write_streamed_rsa_data(struct hfi1_devdata *dd, int what,
805 const u8 *data, int nbytes)
806 {
807 u64 *ptr = (u64 *)data;
808 int qw_size = nbytes / 8;
809
810 for (; qw_size > 0; qw_size--, ptr++)
811 write_csr(dd, what, *ptr);
812 }
813
814 /*
815 * Download the signature and start the RSA mechanism. Wait for
816 * RSA_ENGINE_TIMEOUT before giving up.
817 */
run_rsa(struct hfi1_devdata * dd,const char * who,const u8 * signature)818 static int run_rsa(struct hfi1_devdata *dd, const char *who,
819 const u8 *signature)
820 {
821 unsigned long timeout;
822 u64 reg;
823 u32 status;
824 int ret = 0;
825
826 /* write the signature */
827 write_rsa_data(dd, MISC_CFG_RSA_SIGNATURE, signature, KEY_SIZE);
828
829 /* initialize RSA */
830 write_csr(dd, MISC_CFG_RSA_CMD, RSA_CMD_INIT);
831
832 /*
833 * Make sure the engine is idle and insert a delay between the two
834 * writes to MISC_CFG_RSA_CMD.
835 */
836 status = (read_csr(dd, MISC_CFG_FW_CTRL)
837 & MISC_CFG_FW_CTRL_RSA_STATUS_SMASK)
838 >> MISC_CFG_FW_CTRL_RSA_STATUS_SHIFT;
839 if (status != RSA_STATUS_IDLE) {
840 dd_dev_err(dd, "%s security engine not idle - giving up\n",
841 who);
842 return -EBUSY;
843 }
844
845 /* start RSA */
846 write_csr(dd, MISC_CFG_RSA_CMD, RSA_CMD_START);
847
848 /*
849 * Look for the result.
850 *
851 * The RSA engine is hooked up to two MISC errors. The driver
852 * masks these errors as they do not respond to the standard
853 * error "clear down" mechanism. Look for these errors here and
854 * clear them when possible. This routine will exit with the
855 * errors of the current run still set.
856 *
857 * MISC_FW_AUTH_FAILED_ERR
858 * Firmware authorization failed. This can be cleared by
859 * re-initializing the RSA engine, then clearing the status bit.
860 * Do not re-init the RSA angine immediately after a successful
861 * run - this will reset the current authorization.
862 *
863 * MISC_KEY_MISMATCH_ERR
864 * Key does not match. The only way to clear this is to load
865 * a matching key then clear the status bit. If this error
866 * is raised, it will persist outside of this routine until a
867 * matching key is loaded.
868 */
869 timeout = msecs_to_jiffies(RSA_ENGINE_TIMEOUT) + jiffies;
870 while (1) {
871 status = (read_csr(dd, MISC_CFG_FW_CTRL)
872 & MISC_CFG_FW_CTRL_RSA_STATUS_SMASK)
873 >> MISC_CFG_FW_CTRL_RSA_STATUS_SHIFT;
874
875 if (status == RSA_STATUS_IDLE) {
876 /* should not happen */
877 dd_dev_err(dd, "%s firmware security bad idle state\n",
878 who);
879 ret = -EINVAL;
880 break;
881 } else if (status == RSA_STATUS_DONE) {
882 /* finished successfully */
883 break;
884 } else if (status == RSA_STATUS_FAILED) {
885 /* finished unsuccessfully */
886 ret = -EINVAL;
887 break;
888 }
889 /* else still active */
890
891 if (time_after(jiffies, timeout)) {
892 /*
893 * Timed out while active. We can't reset the engine
894 * if it is stuck active, but run through the
895 * error code to see what error bits are set.
896 */
897 dd_dev_err(dd, "%s firmware security time out\n", who);
898 ret = -ETIMEDOUT;
899 break;
900 }
901
902 msleep(20);
903 }
904
905 /*
906 * Arrive here on success or failure. Clear all RSA engine
907 * errors. All current errors will stick - the RSA logic is keeping
908 * error high. All previous errors will clear - the RSA logic
909 * is not keeping the error high.
910 */
911 write_csr(dd, MISC_ERR_CLEAR,
912 MISC_ERR_STATUS_MISC_FW_AUTH_FAILED_ERR_SMASK |
913 MISC_ERR_STATUS_MISC_KEY_MISMATCH_ERR_SMASK);
914 /*
915 * All that is left are the current errors. Print warnings on
916 * authorization failure details, if any. Firmware authorization
917 * can be retried, so these are only warnings.
918 */
919 reg = read_csr(dd, MISC_ERR_STATUS);
920 if (ret) {
921 if (reg & MISC_ERR_STATUS_MISC_FW_AUTH_FAILED_ERR_SMASK)
922 dd_dev_warn(dd, "%s firmware authorization failed\n",
923 who);
924 if (reg & MISC_ERR_STATUS_MISC_KEY_MISMATCH_ERR_SMASK)
925 dd_dev_warn(dd, "%s firmware key mismatch\n", who);
926 }
927
928 return ret;
929 }
930
load_security_variables(struct hfi1_devdata * dd,struct firmware_details * fdet)931 static void load_security_variables(struct hfi1_devdata *dd,
932 struct firmware_details *fdet)
933 {
934 /* Security variables a. Write the modulus */
935 write_rsa_data(dd, MISC_CFG_RSA_MODULUS, fdet->modulus, KEY_SIZE);
936 /* Security variables b. Write the r2 */
937 write_rsa_data(dd, MISC_CFG_RSA_R2, fdet->r2, KEY_SIZE);
938 /* Security variables c. Write the mu */
939 write_rsa_data(dd, MISC_CFG_RSA_MU, fdet->mu, MU_SIZE);
940 /* Security variables d. Write the header */
941 write_streamed_rsa_data(dd, MISC_CFG_SHA_PRELOAD,
942 (u8 *)fdet->css_header,
943 sizeof(struct css_header));
944 }
945
946 /* return the 8051 firmware state */
get_firmware_state(struct hfi1_devdata * dd)947 static inline u32 get_firmware_state(struct hfi1_devdata *dd)
948 {
949 u64 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
950
951 return (reg >> DC_DC8051_STS_CUR_STATE_FIRMWARE_SHIFT)
952 & DC_DC8051_STS_CUR_STATE_FIRMWARE_MASK;
953 }
954
955 /*
956 * Wait until the firmware is up and ready to take host requests.
957 * Return 0 on success, -ETIMEDOUT on timeout.
958 */
wait_fm_ready(struct hfi1_devdata * dd,u32 mstimeout)959 int wait_fm_ready(struct hfi1_devdata *dd, u32 mstimeout)
960 {
961 unsigned long timeout;
962
963 /* in the simulator, the fake 8051 is always ready */
964 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
965 return 0;
966
967 timeout = msecs_to_jiffies(mstimeout) + jiffies;
968 while (1) {
969 if (get_firmware_state(dd) == 0xa0) /* ready */
970 return 0;
971 if (time_after(jiffies, timeout)) /* timed out */
972 return -ETIMEDOUT;
973 usleep_range(1950, 2050); /* sleep 2ms-ish */
974 }
975 }
976
977 /*
978 * Load the 8051 firmware.
979 */
load_8051_firmware(struct hfi1_devdata * dd,struct firmware_details * fdet)980 static int load_8051_firmware(struct hfi1_devdata *dd,
981 struct firmware_details *fdet)
982 {
983 u64 reg;
984 int ret;
985 u8 ver_major;
986 u8 ver_minor;
987 u8 ver_patch;
988
989 /*
990 * DC Reset sequence
991 * Load DC 8051 firmware
992 */
993 /*
994 * DC reset step 1: Reset DC8051
995 */
996 reg = DC_DC8051_CFG_RST_M8051W_SMASK
997 | DC_DC8051_CFG_RST_CRAM_SMASK
998 | DC_DC8051_CFG_RST_DRAM_SMASK
999 | DC_DC8051_CFG_RST_IRAM_SMASK
1000 | DC_DC8051_CFG_RST_SFR_SMASK;
1001 write_csr(dd, DC_DC8051_CFG_RST, reg);
1002
1003 /*
1004 * DC reset step 2 (optional): Load 8051 data memory with link
1005 * configuration
1006 */
1007
1008 /*
1009 * DC reset step 3: Load DC8051 firmware
1010 */
1011 /* release all but the core reset */
1012 reg = DC_DC8051_CFG_RST_M8051W_SMASK;
1013 write_csr(dd, DC_DC8051_CFG_RST, reg);
1014
1015 /* Firmware load step 1 */
1016 load_security_variables(dd, fdet);
1017
1018 /*
1019 * Firmware load step 2. Clear MISC_CFG_FW_CTRL.FW_8051_LOADED
1020 */
1021 write_csr(dd, MISC_CFG_FW_CTRL, 0);
1022
1023 /* Firmware load steps 3-5 */
1024 ret = write_8051(dd, 1/*code*/, 0, fdet->firmware_ptr,
1025 fdet->firmware_len);
1026 if (ret)
1027 return ret;
1028
1029 /*
1030 * DC reset step 4. Host starts the DC8051 firmware
1031 */
1032 /*
1033 * Firmware load step 6. Set MISC_CFG_FW_CTRL.FW_8051_LOADED
1034 */
1035 write_csr(dd, MISC_CFG_FW_CTRL, MISC_CFG_FW_CTRL_FW_8051_LOADED_SMASK);
1036
1037 /* Firmware load steps 7-10 */
1038 ret = run_rsa(dd, "8051", fdet->signature);
1039 if (ret)
1040 return ret;
1041
1042 /* clear all reset bits, releasing the 8051 */
1043 write_csr(dd, DC_DC8051_CFG_RST, 0ull);
1044
1045 /*
1046 * DC reset step 5. Wait for firmware to be ready to accept host
1047 * requests.
1048 */
1049 ret = wait_fm_ready(dd, TIMEOUT_8051_START);
1050 if (ret) { /* timed out */
1051 dd_dev_err(dd, "8051 start timeout, current state 0x%x\n",
1052 get_firmware_state(dd));
1053 return -ETIMEDOUT;
1054 }
1055
1056 read_misc_status(dd, &ver_major, &ver_minor, &ver_patch);
1057 dd_dev_info(dd, "8051 firmware version %d.%d.%d\n",
1058 (int)ver_major, (int)ver_minor, (int)ver_patch);
1059 dd->dc8051_ver = dc8051_ver(ver_major, ver_minor, ver_patch);
1060 ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION);
1061 if (ret != HCMD_SUCCESS) {
1062 dd_dev_err(dd,
1063 "Failed to set host interface version, return 0x%x\n",
1064 ret);
1065 return -EIO;
1066 }
1067
1068 return 0;
1069 }
1070
1071 /*
1072 * Write the SBus request register
1073 *
1074 * No need for masking - the arguments are sized exactly.
1075 */
sbus_request(struct hfi1_devdata * dd,u8 receiver_addr,u8 data_addr,u8 command,u32 data_in)1076 void sbus_request(struct hfi1_devdata *dd,
1077 u8 receiver_addr, u8 data_addr, u8 command, u32 data_in)
1078 {
1079 write_csr(dd, ASIC_CFG_SBUS_REQUEST,
1080 ((u64)data_in << ASIC_CFG_SBUS_REQUEST_DATA_IN_SHIFT) |
1081 ((u64)command << ASIC_CFG_SBUS_REQUEST_COMMAND_SHIFT) |
1082 ((u64)data_addr << ASIC_CFG_SBUS_REQUEST_DATA_ADDR_SHIFT) |
1083 ((u64)receiver_addr <<
1084 ASIC_CFG_SBUS_REQUEST_RECEIVER_ADDR_SHIFT));
1085 }
1086
1087 /*
1088 * Read a value from the SBus.
1089 *
1090 * Requires the caller to be in fast mode
1091 */
sbus_read(struct hfi1_devdata * dd,u8 receiver_addr,u8 data_addr,u32 data_in)1092 static u32 sbus_read(struct hfi1_devdata *dd, u8 receiver_addr, u8 data_addr,
1093 u32 data_in)
1094 {
1095 u64 reg;
1096 int retries;
1097 int success = 0;
1098 u32 result = 0;
1099 u32 result_code = 0;
1100
1101 sbus_request(dd, receiver_addr, data_addr, READ_SBUS_RECEIVER, data_in);
1102
1103 for (retries = 0; retries < 100; retries++) {
1104 usleep_range(1000, 1200); /* arbitrary */
1105 reg = read_csr(dd, ASIC_STS_SBUS_RESULT);
1106 result_code = (reg >> ASIC_STS_SBUS_RESULT_RESULT_CODE_SHIFT)
1107 & ASIC_STS_SBUS_RESULT_RESULT_CODE_MASK;
1108 if (result_code != SBUS_READ_COMPLETE)
1109 continue;
1110
1111 success = 1;
1112 result = (reg >> ASIC_STS_SBUS_RESULT_DATA_OUT_SHIFT)
1113 & ASIC_STS_SBUS_RESULT_DATA_OUT_MASK;
1114 break;
1115 }
1116
1117 if (!success) {
1118 dd_dev_err(dd, "%s: read failed, result code 0x%x\n", __func__,
1119 result_code);
1120 }
1121
1122 return result;
1123 }
1124
1125 /*
1126 * Turn off the SBus and fabric serdes spicos.
1127 *
1128 * + Must be called with Sbus fast mode turned on.
1129 * + Must be called after fabric serdes broadcast is set up.
1130 * + Must be called before the 8051 is loaded - assumes 8051 is not loaded
1131 * when using MISC_CFG_FW_CTRL.
1132 */
turn_off_spicos(struct hfi1_devdata * dd,int flags)1133 static void turn_off_spicos(struct hfi1_devdata *dd, int flags)
1134 {
1135 /* only needed on A0 */
1136 if (!is_ax(dd))
1137 return;
1138
1139 dd_dev_info(dd, "Turning off spicos:%s%s\n",
1140 flags & SPICO_SBUS ? " SBus" : "",
1141 flags & SPICO_FABRIC ? " fabric" : "");
1142
1143 write_csr(dd, MISC_CFG_FW_CTRL, ENABLE_SPICO_SMASK);
1144 /* disable SBus spico */
1145 if (flags & SPICO_SBUS)
1146 sbus_request(dd, SBUS_MASTER_BROADCAST, 0x01,
1147 WRITE_SBUS_RECEIVER, 0x00000040);
1148
1149 /* disable the fabric serdes spicos */
1150 if (flags & SPICO_FABRIC)
1151 sbus_request(dd, fabric_serdes_broadcast[dd->hfi1_id],
1152 0x07, WRITE_SBUS_RECEIVER, 0x00000000);
1153 write_csr(dd, MISC_CFG_FW_CTRL, 0);
1154 }
1155
1156 /*
1157 * Reset all of the fabric serdes for this HFI in preparation to take the
1158 * link to Polling.
1159 *
1160 * To do a reset, we need to write to to the serdes registers. Unfortunately,
1161 * the fabric serdes download to the other HFI on the ASIC will have turned
1162 * off the firmware validation on this HFI. This means we can't write to the
1163 * registers to reset the serdes. Work around this by performing a complete
1164 * re-download and validation of the fabric serdes firmware. This, as a
1165 * by-product, will reset the serdes. NOTE: the re-download requires that
1166 * the 8051 be in the Offline state. I.e. not actively trying to use the
1167 * serdes. This routine is called at the point where the link is Offline and
1168 * is getting ready to go to Polling.
1169 */
fabric_serdes_reset(struct hfi1_devdata * dd)1170 void fabric_serdes_reset(struct hfi1_devdata *dd)
1171 {
1172 int ret;
1173
1174 if (!fw_fabric_serdes_load)
1175 return;
1176
1177 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
1178 if (ret) {
1179 dd_dev_err(dd,
1180 "Cannot acquire SBus resource to reset fabric SerDes - perhaps you should reboot\n");
1181 return;
1182 }
1183 set_sbus_fast_mode(dd);
1184
1185 if (is_ax(dd)) {
1186 /* A0 serdes do not work with a re-download */
1187 u8 ra = fabric_serdes_broadcast[dd->hfi1_id];
1188
1189 /* place SerDes in reset and disable SPICO */
1190 sbus_request(dd, ra, 0x07, WRITE_SBUS_RECEIVER, 0x00000011);
1191 /* wait 100 refclk cycles @ 156.25MHz => 640ns */
1192 udelay(1);
1193 /* remove SerDes reset */
1194 sbus_request(dd, ra, 0x07, WRITE_SBUS_RECEIVER, 0x00000010);
1195 /* turn SPICO enable on */
1196 sbus_request(dd, ra, 0x07, WRITE_SBUS_RECEIVER, 0x00000002);
1197 } else {
1198 turn_off_spicos(dd, SPICO_FABRIC);
1199 /*
1200 * No need for firmware retry - what to download has already
1201 * been decided.
1202 * No need to pay attention to the load return - the only
1203 * failure is a validation failure, which has already been
1204 * checked by the initial download.
1205 */
1206 (void)load_fabric_serdes_firmware(dd, &fw_fabric);
1207 }
1208
1209 clear_sbus_fast_mode(dd);
1210 release_chip_resource(dd, CR_SBUS);
1211 }
1212
1213 /* Access to the SBus in this routine should probably be serialized */
sbus_request_slow(struct hfi1_devdata * dd,u8 receiver_addr,u8 data_addr,u8 command,u32 data_in)1214 int sbus_request_slow(struct hfi1_devdata *dd,
1215 u8 receiver_addr, u8 data_addr, u8 command, u32 data_in)
1216 {
1217 u64 reg, count = 0;
1218
1219 /* make sure fast mode is clear */
1220 clear_sbus_fast_mode(dd);
1221
1222 sbus_request(dd, receiver_addr, data_addr, command, data_in);
1223 write_csr(dd, ASIC_CFG_SBUS_EXECUTE,
1224 ASIC_CFG_SBUS_EXECUTE_EXECUTE_SMASK);
1225 /* Wait for both DONE and RCV_DATA_VALID to go high */
1226 reg = read_csr(dd, ASIC_STS_SBUS_RESULT);
1227 while (!((reg & ASIC_STS_SBUS_RESULT_DONE_SMASK) &&
1228 (reg & ASIC_STS_SBUS_RESULT_RCV_DATA_VALID_SMASK))) {
1229 if (count++ >= SBUS_MAX_POLL_COUNT) {
1230 u64 counts = read_csr(dd, ASIC_STS_SBUS_COUNTERS);
1231 /*
1232 * If the loop has timed out, we are OK if DONE bit
1233 * is set and RCV_DATA_VALID and EXECUTE counters
1234 * are the same. If not, we cannot proceed.
1235 */
1236 if ((reg & ASIC_STS_SBUS_RESULT_DONE_SMASK) &&
1237 (SBUS_COUNTER(counts, RCV_DATA_VALID) ==
1238 SBUS_COUNTER(counts, EXECUTE)))
1239 break;
1240 return -ETIMEDOUT;
1241 }
1242 udelay(1);
1243 reg = read_csr(dd, ASIC_STS_SBUS_RESULT);
1244 }
1245 count = 0;
1246 write_csr(dd, ASIC_CFG_SBUS_EXECUTE, 0);
1247 /* Wait for DONE to clear after EXECUTE is cleared */
1248 reg = read_csr(dd, ASIC_STS_SBUS_RESULT);
1249 while (reg & ASIC_STS_SBUS_RESULT_DONE_SMASK) {
1250 if (count++ >= SBUS_MAX_POLL_COUNT)
1251 return -ETIME;
1252 udelay(1);
1253 reg = read_csr(dd, ASIC_STS_SBUS_RESULT);
1254 }
1255 return 0;
1256 }
1257
load_fabric_serdes_firmware(struct hfi1_devdata * dd,struct firmware_details * fdet)1258 static int load_fabric_serdes_firmware(struct hfi1_devdata *dd,
1259 struct firmware_details *fdet)
1260 {
1261 int i, err;
1262 const u8 ra = fabric_serdes_broadcast[dd->hfi1_id]; /* receiver addr */
1263
1264 dd_dev_info(dd, "Downloading fabric firmware\n");
1265
1266 /* step 1: load security variables */
1267 load_security_variables(dd, fdet);
1268 /* step 2: place SerDes in reset and disable SPICO */
1269 sbus_request(dd, ra, 0x07, WRITE_SBUS_RECEIVER, 0x00000011);
1270 /* wait 100 refclk cycles @ 156.25MHz => 640ns */
1271 udelay(1);
1272 /* step 3: remove SerDes reset */
1273 sbus_request(dd, ra, 0x07, WRITE_SBUS_RECEIVER, 0x00000010);
1274 /* step 4: assert IMEM override */
1275 sbus_request(dd, ra, 0x00, WRITE_SBUS_RECEIVER, 0x40000000);
1276 /* step 5: download SerDes machine code */
1277 for (i = 0; i < fdet->firmware_len; i += 4) {
1278 sbus_request(dd, ra, 0x0a, WRITE_SBUS_RECEIVER,
1279 *(u32 *)&fdet->firmware_ptr[i]);
1280 }
1281 /* step 6: IMEM override off */
1282 sbus_request(dd, ra, 0x00, WRITE_SBUS_RECEIVER, 0x00000000);
1283 /* step 7: turn ECC on */
1284 sbus_request(dd, ra, 0x0b, WRITE_SBUS_RECEIVER, 0x000c0000);
1285
1286 /* steps 8-11: run the RSA engine */
1287 err = run_rsa(dd, "fabric serdes", fdet->signature);
1288 if (err)
1289 return err;
1290
1291 /* step 12: turn SPICO enable on */
1292 sbus_request(dd, ra, 0x07, WRITE_SBUS_RECEIVER, 0x00000002);
1293 /* step 13: enable core hardware interrupts */
1294 sbus_request(dd, ra, 0x08, WRITE_SBUS_RECEIVER, 0x00000000);
1295
1296 return 0;
1297 }
1298
load_sbus_firmware(struct hfi1_devdata * dd,struct firmware_details * fdet)1299 static int load_sbus_firmware(struct hfi1_devdata *dd,
1300 struct firmware_details *fdet)
1301 {
1302 int i, err;
1303 const u8 ra = SBUS_MASTER_BROADCAST; /* receiver address */
1304
1305 dd_dev_info(dd, "Downloading SBus firmware\n");
1306
1307 /* step 1: load security variables */
1308 load_security_variables(dd, fdet);
1309 /* step 2: place SPICO into reset and enable off */
1310 sbus_request(dd, ra, 0x01, WRITE_SBUS_RECEIVER, 0x000000c0);
1311 /* step 3: remove reset, enable off, IMEM_CNTRL_EN on */
1312 sbus_request(dd, ra, 0x01, WRITE_SBUS_RECEIVER, 0x00000240);
1313 /* step 4: set starting IMEM address for burst download */
1314 sbus_request(dd, ra, 0x03, WRITE_SBUS_RECEIVER, 0x80000000);
1315 /* step 5: download the SBus Master machine code */
1316 for (i = 0; i < fdet->firmware_len; i += 4) {
1317 sbus_request(dd, ra, 0x14, WRITE_SBUS_RECEIVER,
1318 *(u32 *)&fdet->firmware_ptr[i]);
1319 }
1320 /* step 6: set IMEM_CNTL_EN off */
1321 sbus_request(dd, ra, 0x01, WRITE_SBUS_RECEIVER, 0x00000040);
1322 /* step 7: turn ECC on */
1323 sbus_request(dd, ra, 0x16, WRITE_SBUS_RECEIVER, 0x000c0000);
1324
1325 /* steps 8-11: run the RSA engine */
1326 err = run_rsa(dd, "SBus", fdet->signature);
1327 if (err)
1328 return err;
1329
1330 /* step 12: set SPICO_ENABLE on */
1331 sbus_request(dd, ra, 0x01, WRITE_SBUS_RECEIVER, 0x00000140);
1332
1333 return 0;
1334 }
1335
load_pcie_serdes_firmware(struct hfi1_devdata * dd,struct firmware_details * fdet)1336 static int load_pcie_serdes_firmware(struct hfi1_devdata *dd,
1337 struct firmware_details *fdet)
1338 {
1339 int i;
1340 const u8 ra = SBUS_MASTER_BROADCAST; /* receiver address */
1341
1342 dd_dev_info(dd, "Downloading PCIe firmware\n");
1343
1344 /* step 1: load security variables */
1345 load_security_variables(dd, fdet);
1346 /* step 2: assert single step (halts the SBus Master spico) */
1347 sbus_request(dd, ra, 0x05, WRITE_SBUS_RECEIVER, 0x00000001);
1348 /* step 3: enable XDMEM access */
1349 sbus_request(dd, ra, 0x01, WRITE_SBUS_RECEIVER, 0x00000d40);
1350 /* step 4: load firmware into SBus Master XDMEM */
1351 /*
1352 * NOTE: the dmem address, write_en, and wdata are all pre-packed,
1353 * we only need to pick up the bytes and write them
1354 */
1355 for (i = 0; i < fdet->firmware_len; i += 4) {
1356 sbus_request(dd, ra, 0x04, WRITE_SBUS_RECEIVER,
1357 *(u32 *)&fdet->firmware_ptr[i]);
1358 }
1359 /* step 5: disable XDMEM access */
1360 sbus_request(dd, ra, 0x01, WRITE_SBUS_RECEIVER, 0x00000140);
1361 /* step 6: allow SBus Spico to run */
1362 sbus_request(dd, ra, 0x05, WRITE_SBUS_RECEIVER, 0x00000000);
1363
1364 /*
1365 * steps 7-11: run RSA, if it succeeds, firmware is available to
1366 * be swapped
1367 */
1368 return run_rsa(dd, "PCIe serdes", fdet->signature);
1369 }
1370
1371 /*
1372 * Set the given broadcast values on the given list of devices.
1373 */
set_serdes_broadcast(struct hfi1_devdata * dd,u8 bg1,u8 bg2,const u8 * addrs,int count)1374 static void set_serdes_broadcast(struct hfi1_devdata *dd, u8 bg1, u8 bg2,
1375 const u8 *addrs, int count)
1376 {
1377 while (--count >= 0) {
1378 /*
1379 * Set BROADCAST_GROUP_1 and BROADCAST_GROUP_2, leave
1380 * defaults for everything else. Do not read-modify-write,
1381 * per instruction from the manufacturer.
1382 *
1383 * Register 0xfd:
1384 * bits what
1385 * ----- ---------------------------------
1386 * 0 IGNORE_BROADCAST (default 0)
1387 * 11:4 BROADCAST_GROUP_1 (default 0xff)
1388 * 23:16 BROADCAST_GROUP_2 (default 0xff)
1389 */
1390 sbus_request(dd, addrs[count], 0xfd, WRITE_SBUS_RECEIVER,
1391 (u32)bg1 << 4 | (u32)bg2 << 16);
1392 }
1393 }
1394
acquire_hw_mutex(struct hfi1_devdata * dd)1395 int acquire_hw_mutex(struct hfi1_devdata *dd)
1396 {
1397 unsigned long timeout;
1398 int try = 0;
1399 u8 mask = 1 << dd->hfi1_id;
1400 u8 user = (u8)read_csr(dd, ASIC_CFG_MUTEX);
1401
1402 if (user == mask) {
1403 dd_dev_info(dd,
1404 "Hardware mutex already acquired, mutex mask %u\n",
1405 (u32)mask);
1406 return 0;
1407 }
1408
1409 retry:
1410 timeout = msecs_to_jiffies(HM_TIMEOUT) + jiffies;
1411 while (1) {
1412 write_csr(dd, ASIC_CFG_MUTEX, mask);
1413 user = (u8)read_csr(dd, ASIC_CFG_MUTEX);
1414 if (user == mask)
1415 return 0; /* success */
1416 if (time_after(jiffies, timeout))
1417 break; /* timed out */
1418 msleep(20);
1419 }
1420
1421 /* timed out */
1422 dd_dev_err(dd,
1423 "Unable to acquire hardware mutex, mutex mask %u, my mask %u (%s)\n",
1424 (u32)user, (u32)mask, (try == 0) ? "retrying" : "giving up");
1425
1426 if (try == 0) {
1427 /* break mutex and retry */
1428 write_csr(dd, ASIC_CFG_MUTEX, 0);
1429 try++;
1430 goto retry;
1431 }
1432
1433 return -EBUSY;
1434 }
1435
release_hw_mutex(struct hfi1_devdata * dd)1436 void release_hw_mutex(struct hfi1_devdata *dd)
1437 {
1438 u8 mask = 1 << dd->hfi1_id;
1439 u8 user = (u8)read_csr(dd, ASIC_CFG_MUTEX);
1440
1441 if (user != mask)
1442 dd_dev_warn(dd,
1443 "Unable to release hardware mutex, mutex mask %u, my mask %u\n",
1444 (u32)user, (u32)mask);
1445 else
1446 write_csr(dd, ASIC_CFG_MUTEX, 0);
1447 }
1448
1449 /* return the given resource bit(s) as a mask for the given HFI */
resource_mask(u32 hfi1_id,u32 resource)1450 static inline u64 resource_mask(u32 hfi1_id, u32 resource)
1451 {
1452 return ((u64)resource) << (hfi1_id ? CR_DYN_SHIFT : 0);
1453 }
1454
fail_mutex_acquire_message(struct hfi1_devdata * dd,const char * func)1455 static void fail_mutex_acquire_message(struct hfi1_devdata *dd,
1456 const char *func)
1457 {
1458 dd_dev_err(dd,
1459 "%s: hardware mutex stuck - suggest rebooting the machine\n",
1460 func);
1461 }
1462
1463 /*
1464 * Acquire access to a chip resource.
1465 *
1466 * Return 0 on success, -EBUSY if resource busy, -EIO if mutex acquire failed.
1467 */
__acquire_chip_resource(struct hfi1_devdata * dd,u32 resource)1468 static int __acquire_chip_resource(struct hfi1_devdata *dd, u32 resource)
1469 {
1470 u64 scratch0, all_bits, my_bit;
1471 int ret;
1472
1473 if (resource & CR_DYN_MASK) {
1474 /* a dynamic resource is in use if either HFI has set the bit */
1475 if (dd->pcidev->device == PCI_DEVICE_ID_INTEL0 &&
1476 (resource & (CR_I2C1 | CR_I2C2))) {
1477 /* discrete devices must serialize across both chains */
1478 all_bits = resource_mask(0, CR_I2C1 | CR_I2C2) |
1479 resource_mask(1, CR_I2C1 | CR_I2C2);
1480 } else {
1481 all_bits = resource_mask(0, resource) |
1482 resource_mask(1, resource);
1483 }
1484 my_bit = resource_mask(dd->hfi1_id, resource);
1485 } else {
1486 /* non-dynamic resources are not split between HFIs */
1487 all_bits = resource;
1488 my_bit = resource;
1489 }
1490
1491 /* lock against other callers within the driver wanting a resource */
1492 mutex_lock(&dd->asic_data->asic_resource_mutex);
1493
1494 ret = acquire_hw_mutex(dd);
1495 if (ret) {
1496 fail_mutex_acquire_message(dd, __func__);
1497 ret = -EIO;
1498 goto done;
1499 }
1500
1501 scratch0 = read_csr(dd, ASIC_CFG_SCRATCH);
1502 if (scratch0 & all_bits) {
1503 ret = -EBUSY;
1504 } else {
1505 write_csr(dd, ASIC_CFG_SCRATCH, scratch0 | my_bit);
1506 /* force write to be visible to other HFI on another OS */
1507 (void)read_csr(dd, ASIC_CFG_SCRATCH);
1508 }
1509
1510 release_hw_mutex(dd);
1511
1512 done:
1513 mutex_unlock(&dd->asic_data->asic_resource_mutex);
1514 return ret;
1515 }
1516
1517 /*
1518 * Acquire access to a chip resource, wait up to mswait milliseconds for
1519 * the resource to become available.
1520 *
1521 * Return 0 on success, -EBUSY if busy (even after wait), -EIO if mutex
1522 * acquire failed.
1523 */
acquire_chip_resource(struct hfi1_devdata * dd,u32 resource,u32 mswait)1524 int acquire_chip_resource(struct hfi1_devdata *dd, u32 resource, u32 mswait)
1525 {
1526 unsigned long timeout;
1527 int ret;
1528
1529 timeout = jiffies + msecs_to_jiffies(mswait);
1530 while (1) {
1531 ret = __acquire_chip_resource(dd, resource);
1532 if (ret != -EBUSY)
1533 return ret;
1534 /* resource is busy, check our timeout */
1535 if (time_after_eq(jiffies, timeout))
1536 return -EBUSY;
1537 usleep_range(80, 120); /* arbitrary delay */
1538 }
1539 }
1540
1541 /*
1542 * Release access to a chip resource
1543 */
release_chip_resource(struct hfi1_devdata * dd,u32 resource)1544 void release_chip_resource(struct hfi1_devdata *dd, u32 resource)
1545 {
1546 u64 scratch0, bit;
1547
1548 /* only dynamic resources should ever be cleared */
1549 if (!(resource & CR_DYN_MASK)) {
1550 dd_dev_err(dd, "%s: invalid resource 0x%x\n", __func__,
1551 resource);
1552 return;
1553 }
1554 bit = resource_mask(dd->hfi1_id, resource);
1555
1556 /* lock against other callers within the driver wanting a resource */
1557 mutex_lock(&dd->asic_data->asic_resource_mutex);
1558
1559 if (acquire_hw_mutex(dd)) {
1560 fail_mutex_acquire_message(dd, __func__);
1561 goto done;
1562 }
1563
1564 scratch0 = read_csr(dd, ASIC_CFG_SCRATCH);
1565 if ((scratch0 & bit) != 0) {
1566 scratch0 &= ~bit;
1567 write_csr(dd, ASIC_CFG_SCRATCH, scratch0);
1568 /* force write to be visible to other HFI on another OS */
1569 (void)read_csr(dd, ASIC_CFG_SCRATCH);
1570 } else {
1571 dd_dev_warn(dd, "%s: id %d, resource 0x%x: bit not set\n",
1572 __func__, dd->hfi1_id, resource);
1573 }
1574
1575 release_hw_mutex(dd);
1576
1577 done:
1578 mutex_unlock(&dd->asic_data->asic_resource_mutex);
1579 }
1580
1581 /*
1582 * Return true if resource is set, false otherwise. Print a warning
1583 * if not set and a function is supplied.
1584 */
check_chip_resource(struct hfi1_devdata * dd,u32 resource,const char * func)1585 bool check_chip_resource(struct hfi1_devdata *dd, u32 resource,
1586 const char *func)
1587 {
1588 u64 scratch0, bit;
1589
1590 if (resource & CR_DYN_MASK)
1591 bit = resource_mask(dd->hfi1_id, resource);
1592 else
1593 bit = resource;
1594
1595 scratch0 = read_csr(dd, ASIC_CFG_SCRATCH);
1596 if ((scratch0 & bit) == 0) {
1597 if (func)
1598 dd_dev_warn(dd,
1599 "%s: id %d, resource 0x%x, not acquired!\n",
1600 func, dd->hfi1_id, resource);
1601 return false;
1602 }
1603 return true;
1604 }
1605
clear_chip_resources(struct hfi1_devdata * dd,const char * func)1606 static void clear_chip_resources(struct hfi1_devdata *dd, const char *func)
1607 {
1608 u64 scratch0;
1609
1610 /* lock against other callers within the driver wanting a resource */
1611 mutex_lock(&dd->asic_data->asic_resource_mutex);
1612
1613 if (acquire_hw_mutex(dd)) {
1614 fail_mutex_acquire_message(dd, func);
1615 goto done;
1616 }
1617
1618 /* clear all dynamic access bits for this HFI */
1619 scratch0 = read_csr(dd, ASIC_CFG_SCRATCH);
1620 scratch0 &= ~resource_mask(dd->hfi1_id, CR_DYN_MASK);
1621 write_csr(dd, ASIC_CFG_SCRATCH, scratch0);
1622 /* force write to be visible to other HFI on another OS */
1623 (void)read_csr(dd, ASIC_CFG_SCRATCH);
1624
1625 release_hw_mutex(dd);
1626
1627 done:
1628 mutex_unlock(&dd->asic_data->asic_resource_mutex);
1629 }
1630
init_chip_resources(struct hfi1_devdata * dd)1631 void init_chip_resources(struct hfi1_devdata *dd)
1632 {
1633 /* clear any holds left by us */
1634 clear_chip_resources(dd, __func__);
1635 }
1636
finish_chip_resources(struct hfi1_devdata * dd)1637 void finish_chip_resources(struct hfi1_devdata *dd)
1638 {
1639 /* clear any holds left by us */
1640 clear_chip_resources(dd, __func__);
1641 }
1642
set_sbus_fast_mode(struct hfi1_devdata * dd)1643 void set_sbus_fast_mode(struct hfi1_devdata *dd)
1644 {
1645 write_csr(dd, ASIC_CFG_SBUS_EXECUTE,
1646 ASIC_CFG_SBUS_EXECUTE_FAST_MODE_SMASK);
1647 }
1648
clear_sbus_fast_mode(struct hfi1_devdata * dd)1649 void clear_sbus_fast_mode(struct hfi1_devdata *dd)
1650 {
1651 u64 reg, count = 0;
1652
1653 reg = read_csr(dd, ASIC_STS_SBUS_COUNTERS);
1654 while (SBUS_COUNTER(reg, EXECUTE) !=
1655 SBUS_COUNTER(reg, RCV_DATA_VALID)) {
1656 if (count++ >= SBUS_MAX_POLL_COUNT)
1657 break;
1658 udelay(1);
1659 reg = read_csr(dd, ASIC_STS_SBUS_COUNTERS);
1660 }
1661 write_csr(dd, ASIC_CFG_SBUS_EXECUTE, 0);
1662 }
1663
load_firmware(struct hfi1_devdata * dd)1664 int load_firmware(struct hfi1_devdata *dd)
1665 {
1666 int ret;
1667
1668 if (fw_fabric_serdes_load) {
1669 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
1670 if (ret)
1671 return ret;
1672
1673 set_sbus_fast_mode(dd);
1674
1675 set_serdes_broadcast(dd, all_fabric_serdes_broadcast,
1676 fabric_serdes_broadcast[dd->hfi1_id],
1677 fabric_serdes_addrs[dd->hfi1_id],
1678 NUM_FABRIC_SERDES);
1679 turn_off_spicos(dd, SPICO_FABRIC);
1680 do {
1681 ret = load_fabric_serdes_firmware(dd, &fw_fabric);
1682 } while (retry_firmware(dd, ret));
1683
1684 clear_sbus_fast_mode(dd);
1685 release_chip_resource(dd, CR_SBUS);
1686 if (ret)
1687 return ret;
1688 }
1689
1690 if (fw_8051_load) {
1691 do {
1692 ret = load_8051_firmware(dd, &fw_8051);
1693 } while (retry_firmware(dd, ret));
1694 if (ret)
1695 return ret;
1696 }
1697
1698 dump_fw_version(dd);
1699 return 0;
1700 }
1701
hfi1_firmware_init(struct hfi1_devdata * dd)1702 int hfi1_firmware_init(struct hfi1_devdata *dd)
1703 {
1704 /* only RTL can use these */
1705 if (dd->icode != ICODE_RTL_SILICON) {
1706 fw_fabric_serdes_load = 0;
1707 fw_pcie_serdes_load = 0;
1708 fw_sbus_load = 0;
1709 }
1710
1711 /* no 8051 or QSFP on simulator */
1712 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
1713 fw_8051_load = 0;
1714
1715 if (!fw_8051_name) {
1716 if (dd->icode == ICODE_RTL_SILICON)
1717 fw_8051_name = DEFAULT_FW_8051_NAME_ASIC;
1718 else
1719 fw_8051_name = DEFAULT_FW_8051_NAME_FPGA;
1720 }
1721 if (!fw_fabric_serdes_name)
1722 fw_fabric_serdes_name = DEFAULT_FW_FABRIC_NAME;
1723 if (!fw_sbus_name)
1724 fw_sbus_name = DEFAULT_FW_SBUS_NAME;
1725 if (!fw_pcie_serdes_name)
1726 fw_pcie_serdes_name = DEFAULT_FW_PCIE_NAME;
1727
1728 return obtain_firmware(dd);
1729 }
1730
1731 /*
1732 * This function is a helper function for parse_platform_config(...) and
1733 * does not check for validity of the platform configuration cache
1734 * (because we know it is invalid as we are building up the cache).
1735 * As such, this should not be called from anywhere other than
1736 * parse_platform_config
1737 */
check_meta_version(struct hfi1_devdata * dd,u32 * system_table)1738 static int check_meta_version(struct hfi1_devdata *dd, u32 *system_table)
1739 {
1740 u32 meta_ver, meta_ver_meta, ver_start, ver_len, mask;
1741 struct platform_config_cache *pcfgcache = &dd->pcfg_cache;
1742
1743 if (!system_table)
1744 return -EINVAL;
1745
1746 meta_ver_meta =
1747 *(pcfgcache->config_tables[PLATFORM_CONFIG_SYSTEM_TABLE].table_metadata
1748 + SYSTEM_TABLE_META_VERSION);
1749
1750 mask = ((1 << METADATA_TABLE_FIELD_START_LEN_BITS) - 1);
1751 ver_start = meta_ver_meta & mask;
1752
1753 meta_ver_meta >>= METADATA_TABLE_FIELD_LEN_SHIFT;
1754
1755 mask = ((1 << METADATA_TABLE_FIELD_LEN_LEN_BITS) - 1);
1756 ver_len = meta_ver_meta & mask;
1757
1758 ver_start /= 8;
1759 meta_ver = *((u8 *)system_table + ver_start) & ((1 << ver_len) - 1);
1760
1761 if (meta_ver < 4) {
1762 dd_dev_info(
1763 dd, "%s:Please update platform config\n", __func__);
1764 return -EINVAL;
1765 }
1766 return 0;
1767 }
1768
parse_platform_config(struct hfi1_devdata * dd)1769 int parse_platform_config(struct hfi1_devdata *dd)
1770 {
1771 struct platform_config_cache *pcfgcache = &dd->pcfg_cache;
1772 struct hfi1_pportdata *ppd = dd->pport;
1773 u32 *ptr = NULL;
1774 u32 header1 = 0, header2 = 0, magic_num = 0, crc = 0, file_length = 0;
1775 u32 record_idx = 0, table_type = 0, table_length_dwords = 0;
1776 int ret = -EINVAL; /* assume failure */
1777
1778 /*
1779 * For integrated devices that did not fall back to the default file,
1780 * the SI tuning information for active channels is acquired from the
1781 * scratch register bitmap, thus there is no platform config to parse.
1782 * Skip parsing in these situations.
1783 */
1784 if (ppd->config_from_scratch)
1785 return 0;
1786
1787 if (!dd->platform_config.data) {
1788 dd_dev_err(dd, "%s: Missing config file\n", __func__);
1789 goto bail;
1790 }
1791 ptr = (u32 *)dd->platform_config.data;
1792
1793 magic_num = *ptr;
1794 ptr++;
1795 if (magic_num != PLATFORM_CONFIG_MAGIC_NUM) {
1796 dd_dev_err(dd, "%s: Bad config file\n", __func__);
1797 goto bail;
1798 }
1799
1800 /* Field is file size in DWORDs */
1801 file_length = (*ptr) * 4;
1802
1803 /*
1804 * Length can't be larger than partition size. Assume platform
1805 * config format version 4 is being used. Interpret the file size
1806 * field as header instead by not moving the pointer.
1807 */
1808 if (file_length > MAX_PLATFORM_CONFIG_FILE_SIZE) {
1809 dd_dev_info(dd,
1810 "%s:File length out of bounds, using alternative format\n",
1811 __func__);
1812 file_length = PLATFORM_CONFIG_FORMAT_4_FILE_SIZE;
1813 } else {
1814 ptr++;
1815 }
1816
1817 if (file_length > dd->platform_config.size) {
1818 dd_dev_info(dd, "%s:File claims to be larger than read size\n",
1819 __func__);
1820 goto bail;
1821 } else if (file_length < dd->platform_config.size) {
1822 dd_dev_info(dd,
1823 "%s:File claims to be smaller than read size, continuing\n",
1824 __func__);
1825 }
1826 /* exactly equal, perfection */
1827
1828 /*
1829 * In both cases where we proceed, using the self-reported file length
1830 * is the safer option. In case of old format a predefined value is
1831 * being used.
1832 */
1833 while (ptr < (u32 *)(dd->platform_config.data + file_length)) {
1834 header1 = *ptr;
1835 header2 = *(ptr + 1);
1836 if (header1 != ~header2) {
1837 dd_dev_err(dd, "%s: Failed validation at offset %ld\n",
1838 __func__, (ptr - (u32 *)
1839 dd->platform_config.data));
1840 goto bail;
1841 }
1842
1843 record_idx = *ptr &
1844 ((1 << PLATFORM_CONFIG_HEADER_RECORD_IDX_LEN_BITS) - 1);
1845
1846 table_length_dwords = (*ptr >>
1847 PLATFORM_CONFIG_HEADER_TABLE_LENGTH_SHIFT) &
1848 ((1 << PLATFORM_CONFIG_HEADER_TABLE_LENGTH_LEN_BITS) - 1);
1849
1850 table_type = (*ptr >> PLATFORM_CONFIG_HEADER_TABLE_TYPE_SHIFT) &
1851 ((1 << PLATFORM_CONFIG_HEADER_TABLE_TYPE_LEN_BITS) - 1);
1852
1853 /* Done with this set of headers */
1854 ptr += 2;
1855
1856 if (record_idx) {
1857 /* data table */
1858 switch (table_type) {
1859 case PLATFORM_CONFIG_SYSTEM_TABLE:
1860 pcfgcache->config_tables[table_type].num_table =
1861 1;
1862 ret = check_meta_version(dd, ptr);
1863 if (ret)
1864 goto bail;
1865 break;
1866 case PLATFORM_CONFIG_PORT_TABLE:
1867 pcfgcache->config_tables[table_type].num_table =
1868 2;
1869 break;
1870 case PLATFORM_CONFIG_RX_PRESET_TABLE:
1871 case PLATFORM_CONFIG_TX_PRESET_TABLE:
1872 case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
1873 case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
1874 pcfgcache->config_tables[table_type].num_table =
1875 table_length_dwords;
1876 break;
1877 default:
1878 dd_dev_err(dd,
1879 "%s: Unknown data table %d, offset %ld\n",
1880 __func__, table_type,
1881 (ptr - (u32 *)
1882 dd->platform_config.data));
1883 goto bail; /* We don't trust this file now */
1884 }
1885 pcfgcache->config_tables[table_type].table = ptr;
1886 } else {
1887 /* metadata table */
1888 switch (table_type) {
1889 case PLATFORM_CONFIG_SYSTEM_TABLE:
1890 case PLATFORM_CONFIG_PORT_TABLE:
1891 case PLATFORM_CONFIG_RX_PRESET_TABLE:
1892 case PLATFORM_CONFIG_TX_PRESET_TABLE:
1893 case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
1894 case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
1895 break;
1896 default:
1897 dd_dev_err(dd,
1898 "%s: Unknown meta table %d, offset %ld\n",
1899 __func__, table_type,
1900 (ptr -
1901 (u32 *)dd->platform_config.data));
1902 goto bail; /* We don't trust this file now */
1903 }
1904 pcfgcache->config_tables[table_type].table_metadata =
1905 ptr;
1906 }
1907
1908 /* Calculate and check table crc */
1909 crc = crc32_le(~(u32)0, (unsigned char const *)ptr,
1910 (table_length_dwords * 4));
1911 crc ^= ~(u32)0;
1912
1913 /* Jump the table */
1914 ptr += table_length_dwords;
1915 if (crc != *ptr) {
1916 dd_dev_err(dd, "%s: Failed CRC check at offset %ld\n",
1917 __func__, (ptr -
1918 (u32 *)dd->platform_config.data));
1919 ret = -EINVAL;
1920 goto bail;
1921 }
1922 /* Jump the CRC DWORD */
1923 ptr++;
1924 }
1925
1926 pcfgcache->cache_valid = 1;
1927 return 0;
1928 bail:
1929 memset(pcfgcache, 0, sizeof(struct platform_config_cache));
1930 return ret;
1931 }
1932
get_integrated_platform_config_field(struct hfi1_devdata * dd,enum platform_config_table_type_encoding table_type,int field_index,u32 * data)1933 static void get_integrated_platform_config_field(
1934 struct hfi1_devdata *dd,
1935 enum platform_config_table_type_encoding table_type,
1936 int field_index, u32 *data)
1937 {
1938 struct hfi1_pportdata *ppd = dd->pport;
1939 u8 *cache = ppd->qsfp_info.cache;
1940 u32 tx_preset = 0;
1941
1942 switch (table_type) {
1943 case PLATFORM_CONFIG_SYSTEM_TABLE:
1944 if (field_index == SYSTEM_TABLE_QSFP_POWER_CLASS_MAX)
1945 *data = ppd->max_power_class;
1946 else if (field_index == SYSTEM_TABLE_QSFP_ATTENUATION_DEFAULT_25G)
1947 *data = ppd->default_atten;
1948 break;
1949 case PLATFORM_CONFIG_PORT_TABLE:
1950 if (field_index == PORT_TABLE_PORT_TYPE)
1951 *data = ppd->port_type;
1952 else if (field_index == PORT_TABLE_LOCAL_ATTEN_25G)
1953 *data = ppd->local_atten;
1954 else if (field_index == PORT_TABLE_REMOTE_ATTEN_25G)
1955 *data = ppd->remote_atten;
1956 break;
1957 case PLATFORM_CONFIG_RX_PRESET_TABLE:
1958 if (field_index == RX_PRESET_TABLE_QSFP_RX_CDR_APPLY)
1959 *data = (ppd->rx_preset & QSFP_RX_CDR_APPLY_SMASK) >>
1960 QSFP_RX_CDR_APPLY_SHIFT;
1961 else if (field_index == RX_PRESET_TABLE_QSFP_RX_EMP_APPLY)
1962 *data = (ppd->rx_preset & QSFP_RX_EMP_APPLY_SMASK) >>
1963 QSFP_RX_EMP_APPLY_SHIFT;
1964 else if (field_index == RX_PRESET_TABLE_QSFP_RX_AMP_APPLY)
1965 *data = (ppd->rx_preset & QSFP_RX_AMP_APPLY_SMASK) >>
1966 QSFP_RX_AMP_APPLY_SHIFT;
1967 else if (field_index == RX_PRESET_TABLE_QSFP_RX_CDR)
1968 *data = (ppd->rx_preset & QSFP_RX_CDR_SMASK) >>
1969 QSFP_RX_CDR_SHIFT;
1970 else if (field_index == RX_PRESET_TABLE_QSFP_RX_EMP)
1971 *data = (ppd->rx_preset & QSFP_RX_EMP_SMASK) >>
1972 QSFP_RX_EMP_SHIFT;
1973 else if (field_index == RX_PRESET_TABLE_QSFP_RX_AMP)
1974 *data = (ppd->rx_preset & QSFP_RX_AMP_SMASK) >>
1975 QSFP_RX_AMP_SHIFT;
1976 break;
1977 case PLATFORM_CONFIG_TX_PRESET_TABLE:
1978 if (cache[QSFP_EQ_INFO_OFFS] & 0x4)
1979 tx_preset = ppd->tx_preset_eq;
1980 else
1981 tx_preset = ppd->tx_preset_noeq;
1982 if (field_index == TX_PRESET_TABLE_PRECUR)
1983 *data = (tx_preset & TX_PRECUR_SMASK) >>
1984 TX_PRECUR_SHIFT;
1985 else if (field_index == TX_PRESET_TABLE_ATTN)
1986 *data = (tx_preset & TX_ATTN_SMASK) >>
1987 TX_ATTN_SHIFT;
1988 else if (field_index == TX_PRESET_TABLE_POSTCUR)
1989 *data = (tx_preset & TX_POSTCUR_SMASK) >>
1990 TX_POSTCUR_SHIFT;
1991 else if (field_index == TX_PRESET_TABLE_QSFP_TX_CDR_APPLY)
1992 *data = (tx_preset & QSFP_TX_CDR_APPLY_SMASK) >>
1993 QSFP_TX_CDR_APPLY_SHIFT;
1994 else if (field_index == TX_PRESET_TABLE_QSFP_TX_EQ_APPLY)
1995 *data = (tx_preset & QSFP_TX_EQ_APPLY_SMASK) >>
1996 QSFP_TX_EQ_APPLY_SHIFT;
1997 else if (field_index == TX_PRESET_TABLE_QSFP_TX_CDR)
1998 *data = (tx_preset & QSFP_TX_CDR_SMASK) >>
1999 QSFP_TX_CDR_SHIFT;
2000 else if (field_index == TX_PRESET_TABLE_QSFP_TX_EQ)
2001 *data = (tx_preset & QSFP_TX_EQ_SMASK) >>
2002 QSFP_TX_EQ_SHIFT;
2003 break;
2004 case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
2005 case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
2006 default:
2007 break;
2008 }
2009 }
2010
get_platform_fw_field_metadata(struct hfi1_devdata * dd,int table,int field,u32 * field_len_bits,u32 * field_start_bits)2011 static int get_platform_fw_field_metadata(struct hfi1_devdata *dd, int table,
2012 int field, u32 *field_len_bits,
2013 u32 *field_start_bits)
2014 {
2015 struct platform_config_cache *pcfgcache = &dd->pcfg_cache;
2016 u32 *src_ptr = NULL;
2017
2018 if (!pcfgcache->cache_valid)
2019 return -EINVAL;
2020
2021 switch (table) {
2022 case PLATFORM_CONFIG_SYSTEM_TABLE:
2023 case PLATFORM_CONFIG_PORT_TABLE:
2024 case PLATFORM_CONFIG_RX_PRESET_TABLE:
2025 case PLATFORM_CONFIG_TX_PRESET_TABLE:
2026 case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
2027 case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
2028 if (field && field < platform_config_table_limits[table])
2029 src_ptr =
2030 pcfgcache->config_tables[table].table_metadata + field;
2031 break;
2032 default:
2033 dd_dev_info(dd, "%s: Unknown table\n", __func__);
2034 break;
2035 }
2036
2037 if (!src_ptr)
2038 return -EINVAL;
2039
2040 if (field_start_bits)
2041 *field_start_bits = *src_ptr &
2042 ((1 << METADATA_TABLE_FIELD_START_LEN_BITS) - 1);
2043
2044 if (field_len_bits)
2045 *field_len_bits = (*src_ptr >> METADATA_TABLE_FIELD_LEN_SHIFT)
2046 & ((1 << METADATA_TABLE_FIELD_LEN_LEN_BITS) - 1);
2047
2048 return 0;
2049 }
2050
2051 /* This is the central interface to getting data out of the platform config
2052 * file. It depends on parse_platform_config() having populated the
2053 * platform_config_cache in hfi1_devdata, and checks the cache_valid member to
2054 * validate the sanity of the cache.
2055 *
2056 * The non-obvious parameters:
2057 * @table_index: Acts as a look up key into which instance of the tables the
2058 * relevant field is fetched from.
2059 *
2060 * This applies to the data tables that have multiple instances. The port table
2061 * is an exception to this rule as each HFI only has one port and thus the
2062 * relevant table can be distinguished by hfi_id.
2063 *
2064 * @data: pointer to memory that will be populated with the field requested.
2065 * @len: length of memory pointed by @data in bytes.
2066 */
get_platform_config_field(struct hfi1_devdata * dd,enum platform_config_table_type_encoding table_type,int table_index,int field_index,u32 * data,u32 len)2067 int get_platform_config_field(struct hfi1_devdata *dd,
2068 enum platform_config_table_type_encoding
2069 table_type, int table_index, int field_index,
2070 u32 *data, u32 len)
2071 {
2072 int ret = 0, wlen = 0, seek = 0;
2073 u32 field_len_bits = 0, field_start_bits = 0, *src_ptr = NULL;
2074 struct platform_config_cache *pcfgcache = &dd->pcfg_cache;
2075 struct hfi1_pportdata *ppd = dd->pport;
2076
2077 if (data)
2078 memset(data, 0, len);
2079 else
2080 return -EINVAL;
2081
2082 if (ppd->config_from_scratch) {
2083 /*
2084 * Use saved configuration from ppd for integrated platforms
2085 */
2086 get_integrated_platform_config_field(dd, table_type,
2087 field_index, data);
2088 return 0;
2089 }
2090
2091 ret = get_platform_fw_field_metadata(dd, table_type, field_index,
2092 &field_len_bits,
2093 &field_start_bits);
2094 if (ret)
2095 return -EINVAL;
2096
2097 /* Convert length to bits */
2098 len *= 8;
2099
2100 /* Our metadata function checked cache_valid and field_index for us */
2101 switch (table_type) {
2102 case PLATFORM_CONFIG_SYSTEM_TABLE:
2103 src_ptr = pcfgcache->config_tables[table_type].table;
2104
2105 if (field_index != SYSTEM_TABLE_QSFP_POWER_CLASS_MAX) {
2106 if (len < field_len_bits)
2107 return -EINVAL;
2108
2109 seek = field_start_bits / 8;
2110 wlen = field_len_bits / 8;
2111
2112 src_ptr = (u32 *)((u8 *)src_ptr + seek);
2113
2114 /*
2115 * We expect the field to be byte aligned and whole byte
2116 * lengths if we are here
2117 */
2118 memcpy(data, src_ptr, wlen);
2119 return 0;
2120 }
2121 break;
2122 case PLATFORM_CONFIG_PORT_TABLE:
2123 /* Port table is 4 DWORDS */
2124 src_ptr = dd->hfi1_id ?
2125 pcfgcache->config_tables[table_type].table + 4 :
2126 pcfgcache->config_tables[table_type].table;
2127 break;
2128 case PLATFORM_CONFIG_RX_PRESET_TABLE:
2129 case PLATFORM_CONFIG_TX_PRESET_TABLE:
2130 case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
2131 case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
2132 src_ptr = pcfgcache->config_tables[table_type].table;
2133
2134 if (table_index <
2135 pcfgcache->config_tables[table_type].num_table)
2136 src_ptr += table_index;
2137 else
2138 src_ptr = NULL;
2139 break;
2140 default:
2141 dd_dev_info(dd, "%s: Unknown table\n", __func__);
2142 break;
2143 }
2144
2145 if (!src_ptr || len < field_len_bits)
2146 return -EINVAL;
2147
2148 src_ptr += (field_start_bits / 32);
2149 *data = (*src_ptr >> (field_start_bits % 32)) &
2150 ((1 << field_len_bits) - 1);
2151
2152 return 0;
2153 }
2154
2155 /*
2156 * Download the firmware needed for the Gen3 PCIe SerDes. An update
2157 * to the SBus firmware is needed before updating the PCIe firmware.
2158 *
2159 * Note: caller must be holding the SBus resource.
2160 */
load_pcie_firmware(struct hfi1_devdata * dd)2161 int load_pcie_firmware(struct hfi1_devdata *dd)
2162 {
2163 int ret = 0;
2164
2165 /* both firmware loads below use the SBus */
2166 set_sbus_fast_mode(dd);
2167
2168 if (fw_sbus_load) {
2169 turn_off_spicos(dd, SPICO_SBUS);
2170 do {
2171 ret = load_sbus_firmware(dd, &fw_sbus);
2172 } while (retry_firmware(dd, ret));
2173 if (ret)
2174 goto done;
2175 }
2176
2177 if (fw_pcie_serdes_load) {
2178 dd_dev_info(dd, "Setting PCIe SerDes broadcast\n");
2179 set_serdes_broadcast(dd, all_pcie_serdes_broadcast,
2180 pcie_serdes_broadcast[dd->hfi1_id],
2181 pcie_serdes_addrs[dd->hfi1_id],
2182 NUM_PCIE_SERDES);
2183 do {
2184 ret = load_pcie_serdes_firmware(dd, &fw_pcie);
2185 } while (retry_firmware(dd, ret));
2186 if (ret)
2187 goto done;
2188 }
2189
2190 done:
2191 clear_sbus_fast_mode(dd);
2192
2193 return ret;
2194 }
2195
2196 /*
2197 * Read the GUID from the hardware, store it in dd.
2198 */
read_guid(struct hfi1_devdata * dd)2199 void read_guid(struct hfi1_devdata *dd)
2200 {
2201 /* Take the DC out of reset to get a valid GUID value */
2202 write_csr(dd, CCE_DC_CTRL, 0);
2203 (void)read_csr(dd, CCE_DC_CTRL);
2204
2205 dd->base_guid = read_csr(dd, DC_DC8051_CFG_LOCAL_GUID);
2206 dd_dev_info(dd, "GUID %llx",
2207 (unsigned long long)dd->base_guid);
2208 }
2209
2210 /* read and display firmware version info */
dump_fw_version(struct hfi1_devdata * dd)2211 static void dump_fw_version(struct hfi1_devdata *dd)
2212 {
2213 u32 pcie_vers[NUM_PCIE_SERDES];
2214 u32 fabric_vers[NUM_FABRIC_SERDES];
2215 u32 sbus_vers;
2216 int i;
2217 int all_same;
2218 int ret;
2219 u8 rcv_addr;
2220
2221 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
2222 if (ret) {
2223 dd_dev_err(dd, "Unable to acquire SBus to read firmware versions\n");
2224 return;
2225 }
2226
2227 /* set fast mode */
2228 set_sbus_fast_mode(dd);
2229
2230 /* read version for SBus Master */
2231 sbus_request(dd, SBUS_MASTER_BROADCAST, 0x02, WRITE_SBUS_RECEIVER, 0);
2232 sbus_request(dd, SBUS_MASTER_BROADCAST, 0x07, WRITE_SBUS_RECEIVER, 0x1);
2233 /* wait for interrupt to be processed */
2234 usleep_range(10000, 11000);
2235 sbus_vers = sbus_read(dd, SBUS_MASTER_BROADCAST, 0x08, 0x1);
2236 dd_dev_info(dd, "SBus Master firmware version 0x%08x\n", sbus_vers);
2237
2238 /* read version for PCIe SerDes */
2239 all_same = 1;
2240 pcie_vers[0] = 0;
2241 for (i = 0; i < NUM_PCIE_SERDES; i++) {
2242 rcv_addr = pcie_serdes_addrs[dd->hfi1_id][i];
2243 sbus_request(dd, rcv_addr, 0x03, WRITE_SBUS_RECEIVER, 0);
2244 /* wait for interrupt to be processed */
2245 usleep_range(10000, 11000);
2246 pcie_vers[i] = sbus_read(dd, rcv_addr, 0x04, 0x0);
2247 if (i > 0 && pcie_vers[0] != pcie_vers[i])
2248 all_same = 0;
2249 }
2250
2251 if (all_same) {
2252 dd_dev_info(dd, "PCIe SerDes firmware version 0x%x\n",
2253 pcie_vers[0]);
2254 } else {
2255 dd_dev_warn(dd, "PCIe SerDes do not have the same firmware version\n");
2256 for (i = 0; i < NUM_PCIE_SERDES; i++) {
2257 dd_dev_info(dd,
2258 "PCIe SerDes lane %d firmware version 0x%x\n",
2259 i, pcie_vers[i]);
2260 }
2261 }
2262
2263 /* read version for fabric SerDes */
2264 all_same = 1;
2265 fabric_vers[0] = 0;
2266 for (i = 0; i < NUM_FABRIC_SERDES; i++) {
2267 rcv_addr = fabric_serdes_addrs[dd->hfi1_id][i];
2268 sbus_request(dd, rcv_addr, 0x03, WRITE_SBUS_RECEIVER, 0);
2269 /* wait for interrupt to be processed */
2270 usleep_range(10000, 11000);
2271 fabric_vers[i] = sbus_read(dd, rcv_addr, 0x04, 0x0);
2272 if (i > 0 && fabric_vers[0] != fabric_vers[i])
2273 all_same = 0;
2274 }
2275
2276 if (all_same) {
2277 dd_dev_info(dd, "Fabric SerDes firmware version 0x%x\n",
2278 fabric_vers[0]);
2279 } else {
2280 dd_dev_warn(dd, "Fabric SerDes do not have the same firmware version\n");
2281 for (i = 0; i < NUM_FABRIC_SERDES; i++) {
2282 dd_dev_info(dd,
2283 "Fabric SerDes lane %d firmware version 0x%x\n",
2284 i, fabric_vers[i]);
2285 }
2286 }
2287
2288 clear_sbus_fast_mode(dd);
2289 release_chip_resource(dd, CR_SBUS);
2290 }
2291