1 /*- 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <linux/random.h> 31 #include <linux/vmalloc.h> 32 #include <linux/hardirq.h> 33 #include <dev/mlx5/driver.h> 34 #include <dev/mlx5/mlx5_ifc.h> 35 #include "mlx5_core.h" 36 37 #define MLX5_HEALTH_POLL_INTERVAL (2 * HZ) 38 #define MAX_MISSES 3 39 40 enum { 41 MLX5_NIC_IFC_FULL = 0, 42 MLX5_NIC_IFC_DISABLED = 1, 43 MLX5_NIC_IFC_NO_DRAM_NIC = 2, 44 MLX5_NIC_IFC_INVALID = 3, 45 }; 46 47 enum { 48 MLX5_DROP_NEW_HEALTH_WORK, 49 MLX5_DROP_NEW_RECOVERY_WORK, 50 }; 51 52 static u8 get_nic_state(struct mlx5_core_dev *dev) 53 { 54 return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3; 55 } 56 57 static void mlx5_trigger_cmd_completions(struct mlx5_core_dev *dev) 58 { 59 unsigned long flags; 60 u64 vector; 61 62 /* wait for pending handlers to complete */ 63 synchronize_irq(dev->priv.msix_arr[MLX5_EQ_VEC_CMD].vector); 64 spin_lock_irqsave(&dev->cmd.alloc_lock, flags); 65 vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1); 66 if (!vector) 67 goto no_trig; 68 69 vector |= MLX5_TRIGGERED_CMD_COMP; 70 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 71 72 mlx5_core_dbg(dev, "vector 0x%jx\n", (uintmax_t)vector); 73 mlx5_cmd_comp_handler(dev, vector); 74 return; 75 76 no_trig: 77 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 78 } 79 80 static int in_fatal(struct mlx5_core_dev *dev) 81 { 82 struct mlx5_core_health *health = &dev->priv.health; 83 struct mlx5_health_buffer __iomem *h = health->health; 84 85 if (get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) 86 return 1; 87 88 if (ioread32be(&h->fw_ver) == 0xffffffff) 89 return 1; 90 91 return 0; 92 } 93 94 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) 95 { 96 mutex_lock(&dev->intf_state_mutex); 97 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 98 goto unlock; 99 return; 100 } 101 102 if (!force) 103 mlx5_core_err(dev, "internal state error detected\n"); 104 if (pci_channel_offline(dev->pdev) || in_fatal(dev) || force) { 105 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 106 mlx5_trigger_cmd_completions(dev); 107 } 108 109 mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0); 110 if (!force) 111 mlx5_core_err(dev, "system error event triggered\n"); 112 113 unlock: 114 mutex_unlock(&dev->intf_state_mutex); 115 } 116 117 static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) 118 { 119 u8 nic_state = get_nic_state(dev); 120 121 switch (nic_state) { 122 case MLX5_NIC_IFC_FULL: 123 mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n"); 124 break; 125 126 case MLX5_NIC_IFC_DISABLED: 127 mlx5_core_warn(dev, "starting teardown\n"); 128 break; 129 130 case MLX5_NIC_IFC_NO_DRAM_NIC: 131 mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n"); 132 break; 133 default: 134 mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n", 135 nic_state); 136 } 137 138 mlx5_disable_device(dev); 139 } 140 141 static void health_recover(struct work_struct *work) 142 { 143 struct mlx5_core_health *health; 144 struct delayed_work *dwork; 145 struct mlx5_core_dev *dev; 146 struct mlx5_priv *priv; 147 u8 nic_state; 148 149 dwork = container_of(work, struct delayed_work, work); 150 health = container_of(dwork, struct mlx5_core_health, recover_work); 151 priv = container_of(health, struct mlx5_priv, health); 152 dev = container_of(priv, struct mlx5_core_dev, priv); 153 154 nic_state = get_nic_state(dev); 155 if (nic_state == MLX5_NIC_IFC_INVALID) { 156 dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n"); 157 return; 158 } 159 160 dev_err(&dev->pdev->dev, "starting health recovery flow\n"); 161 mlx5_recover_device(dev); 162 } 163 164 /* How much time to wait until health resetting the driver (in msecs) */ 165 #define MLX5_RECOVERY_DELAY_MSECS 60000 166 static void health_care(struct work_struct *work) 167 { 168 unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS); 169 struct mlx5_core_health *health; 170 struct mlx5_core_dev *dev; 171 struct mlx5_priv *priv; 172 unsigned long flags; 173 174 health = container_of(work, struct mlx5_core_health, work); 175 priv = container_of(health, struct mlx5_priv, health); 176 dev = container_of(priv, struct mlx5_core_dev, priv); 177 mlx5_core_warn(dev, "handling bad device here\n"); 178 mlx5_handle_bad_state(dev); 179 180 spin_lock_irqsave(&health->wq_lock, flags); 181 if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags)) 182 schedule_delayed_work(&health->recover_work, recover_delay); 183 else 184 dev_err(&dev->pdev->dev, 185 "new health works are not permitted at this stage\n"); 186 spin_unlock_irqrestore(&health->wq_lock, flags); 187 } 188 189 static int get_next_poll_jiffies(void) 190 { 191 unsigned long next; 192 193 get_random_bytes(&next, sizeof(next)); 194 next %= HZ; 195 next += jiffies + MLX5_HEALTH_POLL_INTERVAL; 196 197 return next; 198 } 199 200 void mlx5_trigger_health_work(struct mlx5_core_dev *dev) 201 { 202 struct mlx5_core_health *health = &dev->priv.health; 203 unsigned long flags; 204 205 spin_lock_irqsave(&health->wq_lock, flags); 206 if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) 207 queue_work(health->wq, &health->work); 208 else 209 dev_err(&dev->pdev->dev, 210 "new health works are not permitted at this stage\n"); 211 spin_unlock_irqrestore(&health->wq_lock, flags); 212 } 213 214 static const char *hsynd_str(u8 synd) 215 { 216 switch (synd) { 217 case MLX5_HEALTH_SYNDR_FW_ERR: 218 return "firmware internal error"; 219 case MLX5_HEALTH_SYNDR_IRISC_ERR: 220 return "irisc not responding"; 221 case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR: 222 return "unrecoverable hardware error"; 223 case MLX5_HEALTH_SYNDR_CRC_ERR: 224 return "firmware CRC error"; 225 case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR: 226 return "ICM fetch PCI error"; 227 case MLX5_HEALTH_SYNDR_HW_FTL_ERR: 228 return "HW fatal error\n"; 229 case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR: 230 return "async EQ buffer overrun"; 231 case MLX5_HEALTH_SYNDR_EQ_ERR: 232 return "EQ error"; 233 case MLX5_HEALTH_SYNDR_EQ_INV: 234 return "Invalid EQ referenced"; 235 case MLX5_HEALTH_SYNDR_FFSER_ERR: 236 return "FFSER error"; 237 case MLX5_HEALTH_SYNDR_HIGH_TEMP: 238 return "High temprature"; 239 default: 240 return "unrecognized error"; 241 } 242 } 243 244 static void print_health_info(struct mlx5_core_dev *dev) 245 { 246 struct mlx5_core_health *health = &dev->priv.health; 247 struct mlx5_health_buffer __iomem *h = health->health; 248 char fw_str[18]; 249 u32 fw; 250 int i; 251 252 /* If the syndrom is 0, the device is OK and no need to print buffer */ 253 if (!ioread8(&h->synd)) 254 return; 255 256 for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) 257 printf("mlx5_core: INFO: ""assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i)); 258 259 printf("mlx5_core: INFO: ""assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr)); 260 printf("mlx5_core: INFO: ""assert_callra 0x%08x\n", ioread32be(&h->assert_callra)); 261 snprintf(fw_str, sizeof(fw_str), "%d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); 262 printf("mlx5_core: INFO: ""fw_ver %s\n", fw_str); 263 printf("mlx5_core: INFO: ""hw_id 0x%08x\n", ioread32be(&h->hw_id)); 264 printf("mlx5_core: INFO: ""irisc_index %d\n", ioread8(&h->irisc_index)); 265 printf("mlx5_core: INFO: ""synd 0x%x: %s\n", ioread8(&h->synd), hsynd_str(ioread8(&h->synd))); 266 printf("mlx5_core: INFO: ""ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); 267 fw = ioread32be(&h->fw_ver); 268 printf("mlx5_core: INFO: ""raw fw_ver 0x%08x\n", fw); 269 } 270 271 static void poll_health(unsigned long data) 272 { 273 struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data; 274 struct mlx5_core_health *health = &dev->priv.health; 275 u32 count; 276 277 if (dev->state != MLX5_DEVICE_STATE_UP) 278 return; 279 280 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 281 goto out; 282 283 count = ioread32be(health->health_counter); 284 if (count == health->prev) 285 ++health->miss_counter; 286 else 287 health->miss_counter = 0; 288 289 health->prev = count; 290 if (health->miss_counter == MAX_MISSES) { 291 mlx5_core_err(dev, "device's health compromised - reached miss count\n"); 292 print_health_info(dev); 293 } 294 295 if (in_fatal(dev) && !health->sick) { 296 health->sick = true; 297 print_health_info(dev); 298 mlx5_trigger_health_work(dev); 299 } 300 301 out: 302 mod_timer(&health->timer, get_next_poll_jiffies()); 303 } 304 305 void mlx5_start_health_poll(struct mlx5_core_dev *dev) 306 { 307 struct mlx5_core_health *health = &dev->priv.health; 308 309 init_timer(&health->timer); 310 health->sick = 0; 311 clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 312 clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); 313 health->health = &dev->iseg->health; 314 health->health_counter = &dev->iseg->health_counter; 315 316 setup_timer(&health->timer, poll_health, (unsigned long)dev); 317 mod_timer(&health->timer, 318 round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL)); 319 } 320 321 void mlx5_stop_health_poll(struct mlx5_core_dev *dev) 322 { 323 struct mlx5_core_health *health = &dev->priv.health; 324 325 del_timer_sync(&health->timer); 326 } 327 328 void mlx5_drain_health_wq(struct mlx5_core_dev *dev) 329 { 330 struct mlx5_core_health *health = &dev->priv.health; 331 unsigned long flags; 332 333 spin_lock_irqsave(&health->wq_lock, flags); 334 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 335 set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); 336 spin_unlock_irqrestore(&health->wq_lock, flags); 337 cancel_delayed_work_sync(&health->recover_work); 338 cancel_work_sync(&health->work); 339 } 340 341 void mlx5_drain_health_recovery(struct mlx5_core_dev *dev) 342 { 343 struct mlx5_core_health *health = &dev->priv.health; 344 unsigned long flags; 345 346 spin_lock_irqsave(&health->wq_lock, flags); 347 set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); 348 spin_unlock_irqrestore(&health->wq_lock, flags); 349 cancel_delayed_work_sync(&dev->priv.health.recover_work); 350 } 351 352 void mlx5_health_cleanup(struct mlx5_core_dev *dev) 353 { 354 struct mlx5_core_health *health = &dev->priv.health; 355 356 destroy_workqueue(health->wq); 357 } 358 359 #define HEALTH_NAME "mlx5_health" 360 int mlx5_health_init(struct mlx5_core_dev *dev) 361 { 362 struct mlx5_core_health *health; 363 char *name; 364 int len; 365 366 health = &dev->priv.health; 367 len = strlen(HEALTH_NAME) + strlen(dev_name(&dev->pdev->dev)); 368 name = kmalloc(len + 1, GFP_KERNEL); 369 if (!name) 370 return -ENOMEM; 371 372 snprintf(name, len, "%s:%s", HEALTH_NAME, dev_name(&dev->pdev->dev)); 373 health->wq = create_singlethread_workqueue(name); 374 kfree(name); 375 if (!health->wq) 376 return -ENOMEM; 377 378 spin_lock_init(&health->wq_lock); 379 INIT_WORK(&health->work, health_care); 380 INIT_DELAYED_WORK(&health->recover_work, health_recover); 381 382 return 0; 383 } 384