1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <cma.h> 30 31 #include <strings.h> 32 #include <errno.h> 33 #include <time.h> 34 #include <fm/fmd_api.h> 35 #include <sys/fm/protocol.h> 36 #include <sys/systeminfo.h> 37 #include <sys/utsname.h> 38 39 cma_t cma; 40 41 cma_stats_t cma_stats = { 42 { "cpu_flts", FMD_TYPE_UINT64, "cpu faults resolved" }, 43 { "cpu_fails", FMD_TYPE_UINT64, "cpu faults unresolveable" }, 44 { "cpu_blfails", FMD_TYPE_UINT64, "failed cpu blacklists" }, 45 { "cpu_supp", FMD_TYPE_UINT64, "cpu offlines suppressed" }, 46 { "cpu_blsupp", FMD_TYPE_UINT64, "cpu blacklists suppressed" }, 47 { "page_flts", FMD_TYPE_UINT64, "page faults resolved" }, 48 { "page_fails", FMD_TYPE_UINT64, "page faults unresolveable" }, 49 { "page_supp", FMD_TYPE_UINT64, "page retires suppressed" }, 50 { "page_nonent", FMD_TYPE_UINT64, "retires for non-existent fmris" }, 51 { "page_retmax", FMD_TYPE_UINT64, "hit max retries for page retire" }, 52 { "bad_flts", FMD_TYPE_UINT64, "invalid fault events received" }, 53 { "nop_flts", FMD_TYPE_UINT64, "inapplicable fault events received" }, 54 { "auto_flts", FMD_TYPE_UINT64, "auto-close faults received" } 55 }; 56 57 typedef struct cma_subscriber { 58 const char *subr_class; 59 const char *subr_sname; 60 uint_t subr_svers; 61 int (*subr_func)(fmd_hdl_t *, nvlist_t *, nvlist_t *, const char *); 62 } cma_subscriber_t; 63 64 static const cma_subscriber_t cma_subrs[] = { 65 { "fault.memory.page", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 66 cma_page_retire }, 67 { "fault.memory.page_sb", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 68 cma_page_retire }, 69 { "fault.memory.page_ck", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 70 cma_page_retire }, 71 { "fault.memory.page_ue", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 72 cma_page_retire }, 73 { "fault.memory.generic-x86.page_ce", FM_FMRI_SCHEME_MEM, 74 FM_MEM_SCHEME_VERSION, cma_page_retire }, 75 { "fault.memory.generic-x86.page_ue", FM_FMRI_SCHEME_MEM, 76 FM_MEM_SCHEME_VERSION, cma_page_retire }, 77 { "fault.memory.intel.page_ce", FM_FMRI_SCHEME_MEM, 78 FM_MEM_SCHEME_VERSION, cma_page_retire }, 79 { "fault.memory.intel.page_ue", FM_FMRI_SCHEME_MEM, 80 FM_MEM_SCHEME_VERSION, cma_page_retire }, 81 { "fault.memory.dimm", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 82 NULL }, 83 { "fault.memory.dimm_sb", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 84 NULL }, 85 { "fault.memory.dimm_ck", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 86 NULL }, 87 { "fault.memory.dimm_ue", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 88 NULL }, 89 { "fault.memory.generic-x86.dimm_ce", FM_FMRI_SCHEME_MEM, 90 FM_MEM_SCHEME_VERSION, NULL }, 91 { "fault.memory.generic-x86.dimm_ue", FM_FMRI_SCHEME_MEM, 92 FM_MEM_SCHEME_VERSION, NULL }, 93 { "fault.memory.intel.dimm_ce", FM_FMRI_SCHEME_MEM, 94 FM_MEM_SCHEME_VERSION, NULL }, 95 { "fault.memory.intel.dimm_ue", FM_FMRI_SCHEME_MEM, 96 FM_MEM_SCHEME_VERSION, NULL }, 97 { "fault.memory.intel.fbd.*", FM_FMRI_SCHEME_HC, 98 FM_HC_SCHEME_VERSION, NULL }, 99 { "fault.memory.dimm_testfail", FM_FMRI_SCHEME_MEM, 100 FM_MEM_SCHEME_VERSION, NULL }, 101 { "fault.memory.bank", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 102 NULL }, 103 { "fault.memory.datapath", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, 104 NULL }, 105 106 /* 107 * The following faults do NOT retire a cpu thread, 108 * and therefore must be intercepted before 109 * the default "fault.cpu.*" dispatch to cma_cpu_retire. 110 */ 111 { "fault.cpu.ultraSPARC-T1.freg", FM_FMRI_SCHEME_CPU, 112 FM_CPU_SCHEME_VERSION, NULL }, 113 { "fault.cpu.ultraSPARC-T1.l2cachedata", FM_FMRI_SCHEME_CPU, 114 FM_CPU_SCHEME_VERSION, NULL }, 115 { "fault.cpu.ultraSPARC-T1.l2cachetag", FM_FMRI_SCHEME_CPU, 116 FM_CPU_SCHEME_VERSION, NULL }, 117 { "fault.cpu.ultraSPARC-T1.l2cachectl", FM_FMRI_SCHEME_CPU, 118 FM_CPU_SCHEME_VERSION, NULL }, 119 { "fault.cpu.ultraSPARC-T1.mau", FM_FMRI_SCHEME_CPU, 120 FM_CPU_SCHEME_VERSION, NULL }, 121 { "fault.cpu.amd.dramchannel", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, 122 NULL }, 123 { "fault.cpu.generic-x86.bus_interconnect_memory", FM_FMRI_SCHEME_CPU, 124 FM_CPU_SCHEME_VERSION, NULL }, 125 { "fault.cpu.generic-x86.bus_interconnect_io", FM_FMRI_SCHEME_CPU, 126 FM_CPU_SCHEME_VERSION, NULL }, 127 { "fault.cpu.generic-x86.bus_interconnect", FM_FMRI_SCHEME_CPU, 128 FM_CPU_SCHEME_VERSION, NULL }, 129 { "fault.cpu.intel.bus_interconnect_memory", FM_FMRI_SCHEME_CPU, 130 FM_CPU_SCHEME_VERSION, NULL }, 131 { "fault.cpu.intel.bus_interconnect_io", FM_FMRI_SCHEME_CPU, 132 FM_CPU_SCHEME_VERSION, NULL }, 133 { "fault.cpu.intel.bus_interconnect", FM_FMRI_SCHEME_CPU, 134 FM_CPU_SCHEME_VERSION, NULL }, 135 { "fault.cpu.intel.nb.*", FM_FMRI_SCHEME_HC, 136 FM_HC_SCHEME_VERSION, NULL }, 137 { "fault.cpu.intel.dma", FM_FMRI_SCHEME_HC, 138 FM_HC_SCHEME_VERSION, NULL }, 139 { "fault.cpu.intel.dma", FM_FMRI_SCHEME_CPU, 140 FM_CPU_SCHEME_VERSION, NULL }, 141 /* 142 * Default "fault.cpu.*" for "mem" scheme ASRU dispatch. 143 */ 144 { "fault.cpu.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, 145 cma_cpu_retire }, 146 { NULL, NULL, 0, NULL } 147 }; 148 149 static const cma_subscriber_t * 150 nvl2subr(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t **asrup) 151 { 152 const cma_subscriber_t *sp; 153 nvlist_t *asru; 154 char *scheme; 155 uint8_t version; 156 char *fltclass = "(unknown)"; 157 158 if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &asru) != 0 || 159 nvlist_lookup_string(asru, FM_FMRI_SCHEME, &scheme) != 0 || 160 nvlist_lookup_uint8(asru, FM_VERSION, &version) != 0) { 161 cma_stats.bad_flts.fmds_value.ui64++; 162 return (NULL); 163 } 164 165 for (sp = cma_subrs; sp->subr_class != NULL; sp++) { 166 if (fmd_nvl_class_match(hdl, nvl, sp->subr_class) && 167 strcmp(scheme, sp->subr_sname) == 0 && 168 version <= sp->subr_svers) { 169 *asrup = asru; 170 return (sp); 171 } 172 } 173 174 (void) nvlist_lookup_string(nvl, FM_CLASS, &fltclass); 175 fmd_hdl_error(hdl, "No handling disposition for %s with asru in " 176 "scheme \"%s\"\n", fltclass, scheme); 177 cma_stats.nop_flts.fmds_value.ui64++; 178 return (NULL); 179 } 180 181 static void 182 cma_recv_list(fmd_hdl_t *hdl, nvlist_t *nvl) 183 { 184 char *uuid = NULL; 185 nvlist_t **nva; 186 uint_t nvc = 0; 187 uint_t keepopen; 188 int err = 0; 189 190 err |= nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid); 191 err |= nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST, 192 &nva, &nvc); 193 if (err != 0) { 194 cma_stats.bad_flts.fmds_value.ui64++; 195 return; 196 } 197 198 keepopen = nvc; 199 while (nvc-- != 0 && !fmd_case_uuclosed(hdl, uuid)) { 200 nvlist_t *nvl = *nva++; 201 const cma_subscriber_t *subr; 202 nvlist_t *asru; 203 204 if ((subr = nvl2subr(hdl, nvl, &asru)) == NULL) 205 continue; 206 207 /* 208 * A handler returns CMA_RA_SUCCESS to indicate that 209 * from this suspects point-of-view the case may be 210 * closed, CMA_RA_FAILURE otherwise. 211 * A handler must not close the case itself. 212 */ 213 if (subr->subr_func != NULL) { 214 err = subr->subr_func(hdl, nvl, asru, uuid); 215 216 if (err == CMA_RA_SUCCESS) 217 keepopen--; 218 } 219 } 220 221 if (!keepopen) 222 fmd_case_uuclose(hdl, uuid); 223 } 224 225 static void 226 cma_recv_one(fmd_hdl_t *hdl, nvlist_t *nvl) 227 { 228 const cma_subscriber_t *subr; 229 nvlist_t *asru; 230 231 if ((subr = nvl2subr(hdl, nvl, &asru)) == NULL) 232 return; 233 234 if (subr->subr_func != NULL) 235 (void) subr->subr_func(hdl, nvl, asru, NULL); 236 237 } 238 239 /*ARGSUSED*/ 240 static void 241 cma_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class) 242 { 243 fmd_hdl_debug(hdl, "received %s\n", class); 244 245 if (strcmp(class, FM_LIST_SUSPECT_CLASS) == 0) 246 cma_recv_list(hdl, nvl); 247 else 248 cma_recv_one(hdl, nvl); 249 } 250 251 /*ARGSUSED*/ 252 static void 253 cma_timeout(fmd_hdl_t *hdl, id_t id, void *arg) 254 { 255 if (id == cma.cma_page_timerid) 256 cma_page_retry(hdl); 257 } 258 259 static const fmd_hdl_ops_t fmd_ops = { 260 cma_recv, /* fmdo_recv */ 261 cma_timeout, /* fmdo_timeout */ 262 NULL, /* fmdo_close */ 263 NULL, /* fmdo_stats */ 264 NULL, /* fmdo_gc */ 265 }; 266 267 static const fmd_prop_t fmd_props[] = { 268 { "cpu_tries", FMD_TYPE_UINT32, "10" }, 269 { "cpu_delay", FMD_TYPE_TIME, "1sec" }, 270 { "cpu_offline_enable", FMD_TYPE_BOOL, "true" }, 271 { "cpu_forced_offline", FMD_TYPE_BOOL, "true" }, 272 { "cpu_blacklist_enable", FMD_TYPE_BOOL, "true" }, 273 { "page_ret_mindelay", FMD_TYPE_TIME, "1sec" }, 274 { "page_ret_maxdelay", FMD_TYPE_TIME, "5min" }, 275 { "page_retire_enable", FMD_TYPE_BOOL, "true" }, 276 #ifdef i386 277 /* 278 * On i386, leaving cases open while we retry the 279 * retire can cause the eft module to use large amounts 280 * of memory. Until eft is fixed, we set a maximum number 281 * of retries on page retires, after which the case will 282 * be closed. 283 */ 284 { "page_retire_maxretries", FMD_TYPE_UINT32, "5" }, 285 #else 286 { "page_retire_maxretries", FMD_TYPE_UINT32, "0" }, 287 #endif /* i386 */ 288 { NULL, 0, NULL } 289 }; 290 291 static const fmd_hdl_info_t fmd_info = { 292 "CPU/Memory Retire Agent", CMA_VERSION, &fmd_ops, fmd_props 293 }; 294 295 void 296 _fmd_init(fmd_hdl_t *hdl) 297 { 298 hrtime_t nsec; 299 char buf[SYS_NMLN]; 300 int ret; 301 302 /* 303 * Abort the cpumem-retire module if Solaris is running under the Xen 304 * hypervisor. 305 */ 306 ret = sysinfo(SI_PLATFORM, buf, sizeof (buf)); 307 if (ret == -1 || (strncmp(buf, "i86xpv", sizeof (buf)) == 0)) 308 return; 309 310 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) 311 return; /* invalid data in configuration file */ 312 313 fmd_hdl_subscribe(hdl, "fault.cpu.*"); 314 fmd_hdl_subscribe(hdl, "fault.memory.*"); 315 316 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (cma_stats) / 317 sizeof (fmd_stat_t), (fmd_stat_t *)&cma_stats); 318 319 cma.cma_cpu_tries = fmd_prop_get_int32(hdl, "cpu_tries"); 320 321 nsec = fmd_prop_get_int64(hdl, "cpu_delay"); 322 cma.cma_cpu_delay.tv_sec = nsec / NANOSEC; 323 cma.cma_cpu_delay.tv_nsec = nsec % NANOSEC; 324 325 cma.cma_page_mindelay = fmd_prop_get_int64(hdl, "page_ret_mindelay"); 326 cma.cma_page_maxdelay = fmd_prop_get_int64(hdl, "page_ret_maxdelay"); 327 328 cma.cma_cpu_dooffline = fmd_prop_get_int32(hdl, "cpu_offline_enable"); 329 cma.cma_cpu_forcedoffline = fmd_prop_get_int32(hdl, 330 "cpu_forced_offline"); 331 cma.cma_cpu_doblacklist = fmd_prop_get_int32(hdl, 332 "cpu_blacklist_enable"); 333 cma.cma_page_doretire = fmd_prop_get_int32(hdl, "page_retire_enable"); 334 cma.cma_page_maxretries = 335 fmd_prop_get_int32(hdl, "page_retire_maxretries"); 336 337 if (cma.cma_page_maxdelay < cma.cma_page_mindelay) 338 fmd_hdl_abort(hdl, "page retirement delays conflict\n"); 339 } 340 341 void 342 _fmd_fini(fmd_hdl_t *hdl) 343 { 344 cma_page_fini(hdl); 345 } 346