1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * Niagara2 Performance Counter Backend
28  */
29 
30 #pragma ident	"%Z%%M%	%I%	%E% SMI"
31 
32 #include <sys/cpuvar.h>
33 #include <sys/systm.h>
34 #include <sys/archsystm.h>
35 #include <sys/cmn_err.h>
36 #include <sys/cpc_impl.h>
37 #include <sys/cpc_pcbe.h>
38 #include <sys/modctl.h>
39 #include <sys/machsystm.h>
40 #include <sys/sdt.h>
41 #include <sys/niagara2regs.h>
42 #include <sys/hsvc.h>
43 #include <sys/hypervisor_api.h>
44 #include <sys/disp.h>
45 
46 static int ni2_pcbe_init(void);
47 static uint_t ni2_pcbe_ncounters(void);
48 static const char *ni2_pcbe_impl_name(void);
49 static const char *ni2_pcbe_cpuref(void);
50 static char *ni2_pcbe_list_events(uint_t picnum);
51 static char *ni2_pcbe_list_attrs(void);
52 static uint64_t ni2_pcbe_event_coverage(char *event);
53 static uint64_t ni2_pcbe_overflow_bitmap(void);
54 static int ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
55     uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
56     void *token);
57 static void ni2_pcbe_program(void *token);
58 static void ni2_pcbe_allstop(void);
59 static void ni2_pcbe_sample(void *token);
60 static void ni2_pcbe_free(void *config);
61 
62 extern void ultra_setpcr(uint64_t);
63 extern uint64_t ultra_getpcr(void);
64 extern void ultra_setpic(uint64_t);
65 extern uint64_t ultra_getpic(void);
66 extern uint64_t ultra_gettick(void);
67 extern char cpu_module_name[];
68 
69 pcbe_ops_t ni2_pcbe_ops = {
70 	PCBE_VER_1,
71 	CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE,
72 	ni2_pcbe_ncounters,
73 	ni2_pcbe_impl_name,
74 	ni2_pcbe_cpuref,
75 	ni2_pcbe_list_events,
76 	ni2_pcbe_list_attrs,
77 	ni2_pcbe_event_coverage,
78 	ni2_pcbe_overflow_bitmap,
79 	ni2_pcbe_configure,
80 	ni2_pcbe_program,
81 	ni2_pcbe_allstop,
82 	ni2_pcbe_sample,
83 	ni2_pcbe_free
84 };
85 
86 typedef struct _ni2_pcbe_config {
87 	uint_t		pcbe_picno;	/* 0 for pic0 or 1 for pic1 */
88 	uint32_t	pcbe_evsel;	/* %pcr event code unshifted */
89 	uint32_t	pcbe_flags;	/* hpriv/user/system/priv */
90 	uint32_t	pcbe_pic;	/* unshifted raw %pic value */
91 } ni2_pcbe_config_t;
92 
93 typedef struct _ni2_event {
94 	const char	*name;
95 	const uint32_t	emask;
96 	const uint32_t	emask_valid;	/* Mask of unreserved MASK bits */
97 } ni2_event_t;
98 
99 #define	ULTRA_PCR_PRIVPIC	(UINT64_C(1) << CPC_NIAGARA2_PCR_PRIV_SHIFT)
100 #define	EV_END {NULL, 0, 0}
101 
102 static const uint64_t	allstopped = (ULTRA_PCR_PRIVPIC |
103 	CPC_NIAGARA2_PCR_HOLDOV0 | CPC_NIAGARA2_PCR_HOLDOV1);
104 
105 /*
106  * We update this array in the program and allstop routine. The array
107  * is checked in the sample routine to allow us to only perform the
108  * PCR.ht bit check when counting is in progress.
109  */
110 static boolean_t ni2_cpc_counting[NCPU];
111 
112 static ni2_event_t ni2_events[] = {
113 	{ "Idle_strands",			0x000, 0x00 },
114 	{ "Br_completed",			0x201, 0xff },
115 	{ "Br_taken",				0x202, 0xff },
116 	{ "Instr_FGU_arithmetic",		0x204, 0xff },
117 	{ "Instr_ld",				0x208, 0xff },
118 	{ "Instr_st",				0x210, 0xff },
119 	{ "Instr_sw",				0x220, 0xff },
120 	{ "Instr_other",			0x240, 0xff },
121 	{ "Atomics",				0x280, 0xff },
122 	{ "Instr_cnt",				0x2fd, 0xff },
123 	{ "IC_miss",				0x301, 0x33 },
124 	{ "DC_miss",				0x302, 0x33 },
125 	{ "L2_imiss",				0x310, 0x33 },
126 	{ "L2_dmiss_ld",			0x320, 0x33 },
127 	{ "ITLB_HWTW_ref_L2",			0x404, 0x3c },
128 	{ "DTLB_HWTW_ref_L2",			0x408, 0x3c },
129 	{ "ITLB_HWTW_miss_L2",			0x410, 0x3c },
130 	{ "DTLB_HWTW_miss_L2",			0x420, 0x3c },
131 	{ "Stream_ld_to_PCX",			0x501, 0x3f },
132 	{ "Stream_st_to_PCX",			0x502, 0x3f },
133 	{ "CPU_ld_to_PCX",			0x504, 0x3f },
134 	{ "CPU_ifetch_to_PCX",			0x508, 0x3f },
135 	{ "CPU_st_to_PCX",			0x510, 0x3f },
136 	{ "MMU_ld_to_PCX",			0x520, 0x3f },
137 	{ "DES_3DES_op",			0x601, 0x3f },
138 	{ "AES_op",				0x602, 0x3f },
139 	{ "RC4_op",				0x604, 0x3f },
140 	{ "MD5_SHA-1_SHA-256_op",		0x608, 0x3f },
141 	{ "MA_op",				0x610, 0x3f },
142 	{ "CRC_TCPIP_cksum",			0x620, 0x3f },
143 	{ "DES_3DES_busy_cycle",		0x701, 0x3f },
144 	{ "AES_busy_cycle",			0x702, 0x3f },
145 	{ "RC4_busy_cycle",			0x704, 0x3f },
146 	{ "MD5_SHA-1_SHA-256_busy_cycle",	0x708, 0x3f },
147 	{ "MA_busy_cycle",			0x710, 0x3f },
148 	{ "CRC_MPA_cksum",			0x720, 0x3f },
149 	{ "ITLB_miss",				0xb04, 0x0c },
150 	{ "DTLB_miss",				0xb08, 0x0c },
151 	{ "TLB_miss",				0xb0c, 0x0c },
152 	EV_END
153 };
154 
155 static const char	*ni2_impl_name = "UltraSPARC T2";
156 static char		*evlist;
157 static size_t		evlist_sz;
158 static uint16_t 	pcr_pic0_mask;
159 static uint16_t 	pcr_pic1_mask;
160 
161 #define	CPU_REF_URL " Documentation for Sun processors can be found at: " \
162 			"http://www.sun.com/processors/manuals"
163 
164 static const char *niagara2_cpuref = "See the \"UltraSPARC T2 User's Manual\" "
165 			"for descriptions of these events." CPU_REF_URL;
166 
167 static boolean_t niagara2_hsvc_available = B_TRUE;
168 
169 static int
170 ni2_pcbe_init(void)
171 {
172 	ni2_event_t	*evp;
173 	int		status;
174 	uint64_t	niagara2_hsvc_major;
175 	uint64_t	niagara2_hsvc_minor;
176 
177 	pcr_pic0_mask = CPC_NIAGARA2_PCR_PIC0_MASK;
178 	pcr_pic1_mask = CPC_NIAGARA2_PCR_PIC1_MASK;
179 
180 	/*
181 	 * Validate API version for Niagara2 specific hypervisor services
182 	 */
183 	status = hsvc_version(HSVC_GROUP_NIAGARA2_CPU, &niagara2_hsvc_major,
184 	    &niagara2_hsvc_minor);
185 	if ((status != 0) || (niagara2_hsvc_major != NIAGARA2_HSVC_MAJOR)) {
186 		cmn_err(CE_WARN, "hypervisor services not negotiated "
187 		    "or unsupported major number: group: 0x%x major: 0x%lx "
188 		    "minor: 0x%lx errno: %d", HSVC_GROUP_NIAGARA2_CPU,
189 		    niagara2_hsvc_major, niagara2_hsvc_minor, status);
190 		niagara2_hsvc_available = B_FALSE;
191 	}
192 
193 	/*
194 	 * Construct event list.
195 	 *
196 	 * First pass:  Calculate size needed. We'll need an additional byte
197 	 *		for the NULL pointer during the last strcat.
198 	 *
199 	 * Second pass: Copy strings.
200 	 */
201 	for (evp = ni2_events; evp->name != NULL; evp++)
202 		evlist_sz += strlen(evp->name) + 1;
203 
204 	evlist = kmem_alloc(evlist_sz + 1, KM_SLEEP);
205 	evlist[0] = '\0';
206 
207 	for (evp = ni2_events; evp->name != NULL; evp++) {
208 		(void) strcat(evlist, evp->name);
209 		(void) strcat(evlist, ",");
210 	}
211 	/*
212 	 * Remove trailing comma.
213 	 */
214 	evlist[evlist_sz - 1] = '\0';
215 
216 	return (0);
217 }
218 
219 static uint_t
220 ni2_pcbe_ncounters(void)
221 {
222 	return (2);
223 }
224 
225 static const char *
226 ni2_pcbe_impl_name(void)
227 {
228 	return (ni2_impl_name);
229 }
230 
231 static const char *
232 ni2_pcbe_cpuref(void)
233 {
234 	return (niagara2_cpuref);
235 }
236 
237 static char *
238 ni2_pcbe_list_events(uint_t picnum)
239 {
240 	ASSERT(picnum < cpc_ncounters);
241 
242 	return (evlist);
243 }
244 
245 static char *
246 ni2_pcbe_list_attrs(void)
247 {
248 	if (niagara2_hsvc_available == B_TRUE)
249 		return ("hpriv,emask");
250 	else
251 		return ("emask");
252 }
253 
254 static ni2_event_t *
255 find_event(char *name)
256 {
257 	ni2_event_t	*evp;
258 
259 	for (evp = ni2_events; evp->name != NULL; evp++)
260 		if (strcmp(name, evp->name) == 0)
261 			return (evp);
262 
263 	return (NULL);
264 }
265 
266 /*ARGSUSED*/
267 static uint64_t
268 ni2_pcbe_event_coverage(char *event)
269 {
270 	/*
271 	 * Fortunately, both pic0 and pic1 can count all events.
272 	 */
273 	return (0x3);
274 }
275 
276 static uint64_t
277 ni2_pcbe_overflow_bitmap(void)
278 {
279 	uint64_t	pcr, overflow;
280 	uint64_t	pic;
281 	uint32_t	pic0, pic1;
282 	boolean_t	update_pic = B_FALSE;
283 
284 	ASSERT(getpil() >= DISP_LEVEL);
285 	pcr = ultra_getpcr();
286 	DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr);
287 	overflow =  (pcr & CPC_NIAGARA2_PCR_OV0_MASK) >>
288 	    CPC_NIAGARA2_PCR_OV0_SHIFT;
289 	overflow |=  (pcr & CPC_NIAGARA2_PCR_OV1_MASK) >>
290 	    CPC_NIAGARA2_PCR_OV1_SHIFT;
291 
292 	pic = ultra_getpic();
293 	pic0 = (uint32_t)(pic & PIC0_MASK);
294 	pic1 = (uint32_t)((pic >> PIC1_SHIFT) & PIC0_MASK);
295 
296 	pcr |= (CPC_NIAGARA2_PCR_HOLDOV0 | CPC_NIAGARA2_PCR_HOLDOV1);
297 
298 	if (overflow & 0x1) {
299 		pcr &= ~(CPC_NIAGARA2_PCR_OV0_MASK |
300 		    CPC_NIAGARA2_PCR_HOLDOV0);
301 		if (PIC_IN_OV_RANGE(pic0)) {
302 			pic0 = 0;
303 			update_pic = B_TRUE;
304 		}
305 	}
306 
307 	if (overflow & 0x2) {
308 		pcr &= ~(CPC_NIAGARA2_PCR_OV1_MASK |
309 		    CPC_NIAGARA2_PCR_HOLDOV1);
310 		if (PIC_IN_OV_RANGE(pic1)) {
311 			pic1 = 0;
312 			update_pic = B_TRUE;
313 		}
314 	}
315 
316 	if (update_pic)
317 		ultra_setpic(((uint64_t)pic1 << PIC1_SHIFT) | pic0);
318 
319 	/*
320 	 * The HV interface does not need to be used here because we are
321 	 * only resetting the OV bits and do not need to set the HT bit.
322 	 */
323 	DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr);
324 	ultra_setpcr(pcr);
325 
326 	return (overflow);
327 }
328 
329 /*ARGSUSED*/
330 static int
331 ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
332     uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
333 {
334 	ni2_pcbe_config_t	*cfg;
335 	ni2_pcbe_config_t	*other_config;
336 	ni2_event_t		*evp;
337 	int			i;
338 	uint32_t		evsel;
339 
340 	/*
341 	 * If we've been handed an existing configuration, we need only preset
342 	 * the counter value.
343 	 */
344 	if (*data != NULL) {
345 		cfg = *data;
346 		cfg->pcbe_pic = (uint32_t)preset;
347 		return (0);
348 	}
349 
350 	if (picnum > 1)
351 		return (CPC_INVALID_PICNUM);
352 
353 	if ((evp = find_event(event)) == NULL)
354 		return (CPC_INVALID_EVENT);
355 
356 	evsel = evp->emask;
357 
358 	for (i = 0; i < nattrs; i++) {
359 		if (strcmp(attrs[i].ka_name, "hpriv") == 0) {
360 			if (attrs[i].ka_val != 0)
361 				flags |= CPC_COUNT_HV;
362 		} else if (strcmp(attrs[i].ka_name, "emask") == 0) {
363 			if ((attrs[i].ka_val | evp->emask_valid) !=
364 			    evp->emask_valid)
365 				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
366 			evsel |= attrs[i].ka_val;
367 		} else
368 			return (CPC_INVALID_ATTRIBUTE);
369 	}
370 
371 	/*
372 	 * Find other requests that will be programmed with this one, and ensure
373 	 * the flags don't conflict.
374 	 */
375 	if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) &&
376 	    (other_config->pcbe_flags != flags))
377 		return (CPC_CONFLICTING_REQS);
378 
379 	/*
380 	 * If the hpriv attribute is present, make sure we have
381 	 * access to hyperprivileged events before continuing with
382 	 * this configuration. If we can set the ht bit in the PCR
383 	 * successfully, we must have access to hyperprivileged
384 	 * events.
385 	 *
386 	 * If this is a static per-CPU configuration, the CPC
387 	 * driver ensures there can not be more than one for this
388 	 * CPU. If this is a per-LWP configuration, the driver
389 	 * ensures no static per-CPU counting is ongoing and that
390 	 * the target LWP is not already being monitored.
391 	 */
392 	if (flags & CPC_COUNT_HV) {
393 		kpreempt_disable();
394 
395 		DTRACE_PROBE1(niagara2__setpcr, uint64_t,
396 		    allstopped | CPC_NIAGARA2_PCR_HT);
397 		if (hv_niagara_setperf(HV_NIAGARA_SPARC_CTL,
398 		    allstopped | CPC_NIAGARA2_PCR_HT) != H_EOK) {
399 			kpreempt_enable();
400 			return (CPC_HV_NO_ACCESS);
401 		}
402 
403 		DTRACE_PROBE1(niagara2__setpcr, uint64_t, allstopped);
404 		(void) hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, allstopped);
405 
406 		kpreempt_enable();
407 	}
408 
409 	cfg = kmem_alloc(sizeof (*cfg), KM_SLEEP);
410 
411 	cfg->pcbe_picno = picnum;
412 	cfg->pcbe_evsel = evsel;
413 	cfg->pcbe_flags = flags;
414 	cfg->pcbe_pic = (uint32_t)preset;
415 
416 	*data = cfg;
417 	return (0);
418 }
419 
420 static void
421 ni2_pcbe_program(void *token)
422 {
423 	ni2_pcbe_config_t	*pic0;
424 	ni2_pcbe_config_t	*pic1;
425 	ni2_pcbe_config_t	*tmp;
426 	ni2_pcbe_config_t	nullcfg = { 1, 0, 0, 0 };
427 	uint64_t		pcr;
428 	uint64_t		curpic;
429 	uint64_t		toe;
430 
431 	/* enable trap-on-event for pic0 and pic1 */
432 	toe = (CPC_NIAGARA2_PCR_TOE0 | CPC_NIAGARA2_PCR_TOE1);
433 
434 	if ((pic0 = (ni2_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) ==
435 	    NULL)
436 		panic("ni2_pcbe: token %p has no configs", token);
437 
438 	if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) {
439 		pic1 = &nullcfg;
440 		nullcfg.pcbe_flags = pic0->pcbe_flags;
441 		toe = CPC_NIAGARA2_PCR_TOE0; /* enable trap-on-event for pic0 */
442 	}
443 
444 	if (pic0->pcbe_picno != 0) {
445 		/*
446 		 * pic0 is counter 1, so if we need the null config it should
447 		 * be counter 0.
448 		 */
449 		nullcfg.pcbe_picno = 0;
450 		tmp = pic0;
451 		pic0 = pic1;
452 		pic1 = tmp;
453 		toe = CPC_NIAGARA2_PCR_TOE1; /* enable trap-on-event for pic1 */
454 	}
455 
456 	if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
457 		panic("%s: bad config on token %p\n", ni2_impl_name, token);
458 
459 	/*
460 	 * UltraSPARC does not allow pic0 to be configured differently
461 	 * from pic1. If the flags on these two configurations are
462 	 * different, they are incompatible. This condition should be
463 	 * caught at configure time.
464 	 */
465 	ASSERT(pic0->pcbe_flags == pic1->pcbe_flags);
466 
467 	ni2_pcbe_allstop();
468 
469 	ultra_setpic(((uint64_t)pic1->pcbe_pic << PIC1_SHIFT) |
470 	    (uint64_t)pic0->pcbe_pic);
471 
472 	pcr = (pic0->pcbe_evsel & pcr_pic0_mask) << CPC_NIAGARA2_PCR_PIC0_SHIFT;
473 	pcr |= (pic1->pcbe_evsel & pcr_pic1_mask) <<
474 	    CPC_NIAGARA2_PCR_PIC1_SHIFT;
475 
476 	if (pic0->pcbe_flags & CPC_COUNT_USER)
477 		pcr |= (1ull << CPC_NIAGARA2_PCR_UT_SHIFT);
478 	if (pic0->pcbe_flags & CPC_COUNT_SYSTEM)
479 		pcr |= (1ull << CPC_NIAGARA2_PCR_ST_SHIFT);
480 	if (pic0->pcbe_flags & CPC_COUNT_HV)
481 		pcr |= (1ull << CPC_NIAGARA2_PCR_HT_SHIFT);
482 	pcr |= toe;
483 
484 	DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr);
485 
486 	if (pic0->pcbe_flags & CPC_COUNT_HV) {
487 		/*
488 		 * The ht bit in the PCR is only writable in
489 		 * hyperprivileged mode. So if we are counting
490 		 * hpriv events, we must use the HV interface
491 		 * hv_niagara_setperf to set the PCR. If this
492 		 * fails, assume we no longer have access to
493 		 * hpriv events.
494 		 */
495 		if (hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, pcr) != H_EOK) {
496 			kcpc_invalidate_config(token);
497 			return;
498 		}
499 	} else
500 		/* Set the PCR with no hpriv event counting enabled. */
501 		ultra_setpcr(pcr);
502 
503 	ni2_cpc_counting[CPU->cpu_id] = B_TRUE;
504 
505 	/*
506 	 * On UltraSPARC, only read-to-read counts are accurate. We cannot
507 	 * expect the value we wrote into the PIC, above, to be there after
508 	 * starting the counter. We must sample the counter value now and use
509 	 * that as the baseline for future samples.
510 	 */
511 	curpic = ultra_getpic();
512 	pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
513 	pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
514 
515 	DTRACE_PROBE1(niagara2__newpic, uint64_t, curpic);
516 }
517 
518 static void
519 ni2_pcbe_allstop(void)
520 {
521 	/*
522 	 * We use the HV interface here because if we were counting
523 	 * hyperprivileged events, we must reset the PCR.ht bit to stop
524 	 * the counting. In the event that this HV call fails, we fall
525 	 * back on ultra_setpcr which does not have write access to the
526 	 * ht bit.
527 	 */
528 	if (hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, allstopped) != H_EOK)
529 		ultra_setpcr(allstopped);
530 
531 	ni2_cpc_counting[CPU->cpu_id] = B_FALSE;
532 }
533 
534 static void
535 ni2_pcbe_sample(void *token)
536 {
537 	uint64_t		curpic;
538 	int64_t			diff;
539 	uint64_t		*pic0_data;
540 	uint64_t		*pic1_data;
541 	uint64_t		*dtmp;
542 	uint64_t		tmp;
543 	uint64_t		pcr;
544 	ni2_pcbe_config_t	*pic0;
545 	ni2_pcbe_config_t	*pic1;
546 	ni2_pcbe_config_t	nullcfg = { 1, 0, 0, 0 };
547 	ni2_pcbe_config_t	*ctmp;
548 
549 	curpic = ultra_getpic();
550 	DTRACE_PROBE1(niagara2__getpic, uint64_t, curpic);
551 
552 	if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL)
553 		panic("%s: token %p has no configs", ni2_impl_name, token);
554 
555 	if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) {
556 		pic1 = &nullcfg;
557 		pic1_data = &tmp;
558 	}
559 
560 	if (pic0->pcbe_picno != 0) {
561 		nullcfg.pcbe_picno = 0;
562 		ctmp = pic0;
563 		pic0 = pic1;
564 		pic1 = ctmp;
565 		dtmp = pic0_data;
566 		pic0_data = pic1_data;
567 		pic1_data = dtmp;
568 	}
569 
570 	if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
571 		panic("%s: bad config on token %p\n", ni2_impl_name, token);
572 
573 
574 	if (pic0->pcbe_flags & CPC_COUNT_HV) {
575 		/*
576 		 * If the hpriv attribute is present, but the HT bit
577 		 * is not set in the PCR, access to hyperprivileged
578 		 * events must have been revoked. Only perform this
579 		 * check if counting is not stopped.
580 		 */
581 		pcr = ultra_getpcr();
582 		DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr);
583 		if (ni2_cpc_counting[CPU->cpu_id] &&
584 		    !(pcr & CPC_NIAGARA2_PCR_HT)) {
585 			kcpc_invalidate_config(token);
586 			return;
587 		}
588 	}
589 
590 	diff = (curpic & PIC0_MASK) - (uint64_t)pic0->pcbe_pic;
591 	if (diff < 0)
592 		diff += (1ll << 32);
593 	*pic0_data += diff;
594 
595 	diff = (curpic >> 32) - (uint64_t)pic1->pcbe_pic;
596 	if (diff < 0)
597 		diff += (1ll << 32);
598 	*pic1_data += diff;
599 
600 	pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
601 	pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
602 }
603 
604 static void
605 ni2_pcbe_free(void *config)
606 {
607 	kmem_free(config, sizeof (ni2_pcbe_config_t));
608 }
609 
610 
611 static struct modlpcbe modlpcbe = {
612 	&mod_pcbeops,
613 	"UltraSPARC T2 Performance Counters v%I%",
614 	&ni2_pcbe_ops
615 };
616 
617 static struct modlinkage modl = {
618 	MODREV_1,
619 	&modlpcbe,
620 };
621 
622 int
623 _init(void)
624 {
625 	if (ni2_pcbe_init() != 0)
626 		return (ENOTSUP);
627 	return (mod_install(&modl));
628 }
629 
630 int
631 _fini(void)
632 {
633 	return (mod_remove(&modl));
634 }
635 
636 int
637 _info(struct modinfo *mi)
638 {
639 	return (mod_info(&modl, mi));
640 }
641