1 /* 2 * Copyright © 2015-2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Robert Bragg <robert@sixbynine.org> 25 */ 26 27 28 /** 29 * DOC: i915 Perf Overview 30 * 31 * Gen graphics supports a large number of performance counters that can help 32 * driver and application developers understand and optimize their use of the 33 * GPU. 34 * 35 * This i915 perf interface enables userspace to configure and open a file 36 * descriptor representing a stream of GPU metrics which can then be read() as 37 * a stream of sample records. 38 * 39 * The interface is particularly suited to exposing buffered metrics that are 40 * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU. 41 * 42 * Streams representing a single context are accessible to applications with a 43 * corresponding drm file descriptor, such that OpenGL can use the interface 44 * without special privileges. Access to system-wide metrics requires root 45 * privileges by default, unless changed via the dev.i915.perf_event_paranoid 46 * sysctl option. 47 * 48 */ 49 50 /** 51 * DOC: i915 Perf History and Comparison with Core Perf 52 * 53 * The interface was initially inspired by the core Perf infrastructure but 54 * some notable differences are: 55 * 56 * i915 perf file descriptors represent a "stream" instead of an "event"; where 57 * a perf event primarily corresponds to a single 64bit value, while a stream 58 * might sample sets of tightly-coupled counters, depending on the 59 * configuration. For example the Gen OA unit isn't designed to support 60 * orthogonal configurations of individual counters; it's configured for a set 61 * of related counters. Samples for an i915 perf stream capturing OA metrics 62 * will include a set of counter values packed in a compact HW specific format. 63 * The OA unit supports a number of different packing formats which can be 64 * selected by the user opening the stream. Perf has support for grouping 65 * events, but each event in the group is configured, validated and 66 * authenticated individually with separate system calls. 67 * 68 * i915 perf stream configurations are provided as an array of u64 (key,value) 69 * pairs, instead of a fixed struct with multiple miscellaneous config members, 70 * interleaved with event-type specific members. 71 * 72 * i915 perf doesn't support exposing metrics via an mmap'd circular buffer. 73 * The supported metrics are being written to memory by the GPU unsynchronized 74 * with the CPU, using HW specific packing formats for counter sets. Sometimes 75 * the constraints on HW configuration require reports to be filtered before it 76 * would be acceptable to expose them to unprivileged applications - to hide 77 * the metrics of other processes/contexts. For these use cases a read() based 78 * interface is a good fit, and provides an opportunity to filter data as it 79 * gets copied from the GPU mapped buffers to userspace buffers. 80 * 81 * 82 * Issues hit with first prototype based on Core Perf 83 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 84 * 85 * The first prototype of this driver was based on the core perf 86 * infrastructure, and while we did make that mostly work, with some changes to 87 * perf, we found we were breaking or working around too many assumptions baked 88 * into perf's currently cpu centric design. 89 * 90 * In the end we didn't see a clear benefit to making perf's implementation and 91 * interface more complex by changing design assumptions while we knew we still 92 * wouldn't be able to use any existing perf based userspace tools. 93 * 94 * Also considering the Gen specific nature of the Observability hardware and 95 * how userspace will sometimes need to combine i915 perf OA metrics with 96 * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're 97 * expecting the interface to be used by a platform specific userspace such as 98 * OpenGL or tools. This is to say; we aren't inherently missing out on having 99 * a standard vendor/architecture agnostic interface by not using perf. 100 * 101 * 102 * For posterity, in case we might re-visit trying to adapt core perf to be 103 * better suited to exposing i915 metrics these were the main pain points we 104 * hit: 105 * 106 * - The perf based OA PMU driver broke some significant design assumptions: 107 * 108 * Existing perf pmus are used for profiling work on a cpu and we were 109 * introducing the idea of _IS_DEVICE pmus with different security 110 * implications, the need to fake cpu-related data (such as user/kernel 111 * registers) to fit with perf's current design, and adding _DEVICE records 112 * as a way to forward device-specific status records. 113 * 114 * The OA unit writes reports of counters into a circular buffer, without 115 * involvement from the CPU, making our PMU driver the first of a kind. 116 * 117 * Given the way we were periodically forward data from the GPU-mapped, OA 118 * buffer to perf's buffer, those bursts of sample writes looked to perf like 119 * we were sampling too fast and so we had to subvert its throttling checks. 120 * 121 * Perf supports groups of counters and allows those to be read via 122 * transactions internally but transactions currently seem designed to be 123 * explicitly initiated from the cpu (say in response to a userspace read()) 124 * and while we could pull a report out of the OA buffer we can't 125 * trigger a report from the cpu on demand. 126 * 127 * Related to being report based; the OA counters are configured in HW as a 128 * set while perf generally expects counter configurations to be orthogonal. 129 * Although counters can be associated with a group leader as they are 130 * opened, there's no clear precedent for being able to provide group-wide 131 * configuration attributes (for example we want to let userspace choose the 132 * OA unit report format used to capture all counters in a set, or specify a 133 * GPU context to filter metrics on). We avoided using perf's grouping 134 * feature and forwarded OA reports to userspace via perf's 'raw' sample 135 * field. This suited our userspace well considering how coupled the counters 136 * are when dealing with normalizing. It would be inconvenient to split 137 * counters up into separate events, only to require userspace to recombine 138 * them. For Mesa it's also convenient to be forwarded raw, periodic reports 139 * for combining with the side-band raw reports it captures using 140 * MI_REPORT_PERF_COUNT commands. 141 * 142 * - As a side note on perf's grouping feature; there was also some concern 143 * that using PERF_FORMAT_GROUP as a way to pack together counter values 144 * would quite drastically inflate our sample sizes, which would likely 145 * lower the effective sampling resolutions we could use when the available 146 * memory bandwidth is limited. 147 * 148 * With the OA unit's report formats, counters are packed together as 32 149 * or 40bit values, with the largest report size being 256 bytes. 150 * 151 * PERF_FORMAT_GROUP values are 64bit, but there doesn't appear to be a 152 * documented ordering to the values, implying PERF_FORMAT_ID must also be 153 * used to add a 64bit ID before each value; giving 16 bytes per counter. 154 * 155 * Related to counter orthogonality; we can't time share the OA unit, while 156 * event scheduling is a central design idea within perf for allowing 157 * userspace to open + enable more events than can be configured in HW at any 158 * one time. The OA unit is not designed to allow re-configuration while in 159 * use. We can't reconfigure the OA unit without losing internal OA unit 160 * state which we can't access explicitly to save and restore. Reconfiguring 161 * the OA unit is also relatively slow, involving ~100 register writes. From 162 * userspace Mesa also depends on a stable OA configuration when emitting 163 * MI_REPORT_PERF_COUNT commands and importantly the OA unit can't be 164 * disabled while there are outstanding MI_RPC commands lest we hang the 165 * command streamer. 166 * 167 * The contents of sample records aren't extensible by device drivers (i.e. 168 * the sample_type bits). As an example; Sourab Gupta had been looking to 169 * attach GPU timestamps to our OA samples. We were shoehorning OA reports 170 * into sample records by using the 'raw' field, but it's tricky to pack more 171 * than one thing into this field because events/core.c currently only lets a 172 * pmu give a single raw data pointer plus len which will be copied into the 173 * ring buffer. To include more than the OA report we'd have to copy the 174 * report into an intermediate larger buffer. I'd been considering allowing a 175 * vector of data+len values to be specified for copying the raw data, but 176 * it felt like a kludge to being using the raw field for this purpose. 177 * 178 * - It felt like our perf based PMU was making some technical compromises 179 * just for the sake of using perf: 180 * 181 * perf_event_open() requires events to either relate to a pid or a specific 182 * cpu core, while our device pmu related to neither. Events opened with a 183 * pid will be automatically enabled/disabled according to the scheduling of 184 * that process - so not appropriate for us. When an event is related to a 185 * cpu id, perf ensures pmu methods will be invoked via an inter process 186 * interrupt on that core. To avoid invasive changes our userspace opened OA 187 * perf events for a specific cpu. This was workable but it meant the 188 * majority of the OA driver ran in atomic context, including all OA report 189 * forwarding, which wasn't really necessary in our case and seems to make 190 * our locking requirements somewhat complex as we handled the interaction 191 * with the rest of the i915 driver. 192 */ 193 194 #include <linux/anon_inodes.h> 195 #include <linux/sizes.h> 196 197 #include "i915_drv.h" 198 #include "i915_oa_hsw.h" 199 200 /* HW requires this to be a power of two, between 128k and 16M, though driver 201 * is currently generally designed assuming the largest 16M size is used such 202 * that the overflow cases are unlikely in normal operation. 203 */ 204 #define OA_BUFFER_SIZE SZ_16M 205 206 #define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1)) 207 208 /* There's a HW race condition between OA unit tail pointer register updates and 209 * writes to memory whereby the tail pointer can sometimes get ahead of what's 210 * been written out to the OA buffer so far. 211 * 212 * Although this can be observed explicitly by checking for a zeroed report-id 213 * field in tail reports, it seems preferable to account for this earlier e.g. 214 * as part of the _oa_buffer_is_empty checks to minimize -EAGAIN polling cycles 215 * in this situation. 216 * 217 * To give time for the most recent reports to land before they may be copied to 218 * userspace, the driver operates as if the tail pointer effectively lags behind 219 * the HW tail pointer by 'tail_margin' bytes. The margin in bytes is calculated 220 * based on this constant in nanoseconds, the current OA sampling exponent 221 * and current report size. 222 * 223 * There is also a fallback check while reading to simply skip over reports with 224 * a zeroed report-id. 225 */ 226 #define OA_TAIL_MARGIN_NSEC 100000ULL 227 228 /* frequency for checking whether the OA unit has written new reports to the 229 * circular OA buffer... 230 */ 231 #define POLL_FREQUENCY 200 232 #define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY) 233 234 #if 0 235 /* for sysctl proc_dointvec_minmax of dev.i915.perf_stream_paranoid */ 236 static int zero; 237 static int one = 1; 238 static u32 i915_perf_stream_paranoid = true; 239 240 /* The maximum exponent the hardware accepts is 63 (essentially it selects one 241 * of the 64bit timestamp bits to trigger reports from) but there's currently 242 * no known use case for sampling as infrequently as once per 47 thousand years. 243 * 244 * Since the timestamps included in OA reports are only 32bits it seems 245 * reasonable to limit the OA exponent where it's still possible to account for 246 * overflow in OA report timestamps. 247 */ 248 #define OA_EXPONENT_MAX 31 249 250 #define INVALID_CTX_ID 0xffffffff 251 252 253 /* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate 254 * 255 * 160ns is the smallest sampling period we can theoretically program the OA 256 * unit with on Haswell, corresponding to 6.25MHz. 257 */ 258 static int oa_sample_rate_hard_limit = 6250000; 259 260 /* Theoretically we can program the OA unit to sample every 160ns but don't 261 * allow that by default unless root... 262 * 263 * The default threshold of 100000Hz is based on perf's similar 264 * kernel.perf_event_max_sample_rate sysctl parameter. 265 */ 266 static u32 i915_oa_max_sample_rate = 100000; 267 268 /* XXX: beware if future OA HW adds new report formats that the current 269 * code assumes all reports have a power-of-two size and ~(size - 1) can 270 * be used as a mask to align the OA tail pointer. 271 */ 272 static struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = { 273 [I915_OA_FORMAT_A13] = { 0, 64 }, 274 [I915_OA_FORMAT_A29] = { 1, 128 }, 275 [I915_OA_FORMAT_A13_B8_C8] = { 2, 128 }, 276 /* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */ 277 [I915_OA_FORMAT_B4_C8] = { 4, 64 }, 278 [I915_OA_FORMAT_A45_B8_C8] = { 5, 256 }, 279 [I915_OA_FORMAT_B4_C8_A16] = { 6, 128 }, 280 [I915_OA_FORMAT_C4_B8] = { 7, 64 }, 281 }; 282 #endif 283 284 #define SAMPLE_OA_REPORT (1<<0) 285 286 /** 287 * struct perf_open_properties - for validated properties given to open a stream 288 * @sample_flags: `DRM_I915_PERF_PROP_SAMPLE_*` properties are tracked as flags 289 * @single_context: Whether a single or all gpu contexts should be monitored 290 * @ctx_handle: A gem ctx handle for use with @single_context 291 * @metrics_set: An ID for an OA unit metric set advertised via sysfs 292 * @oa_format: An OA unit HW report format 293 * @oa_periodic: Whether to enable periodic OA unit sampling 294 * @oa_period_exponent: The OA unit sampling period is derived from this 295 * 296 * As read_properties_unlocked() enumerates and validates the properties given 297 * to open a stream of metrics the configuration is built up in the structure 298 * which starts out zero initialized. 299 */ 300 struct perf_open_properties { 301 u32 sample_flags; 302 303 u64 single_context:1; 304 u64 ctx_handle; 305 306 /* OA sampling state */ 307 int metrics_set; 308 int oa_format; 309 bool oa_periodic; 310 int oa_period_exponent; 311 }; 312 313 #if 0 314 /* NB: This is either called via fops or the poll check hrtimer (atomic ctx) 315 * 316 * It's safe to read OA config state here unlocked, assuming that this is only 317 * called while the stream is enabled, while the global OA configuration can't 318 * be modified. 319 * 320 * Note: we don't lock around the head/tail reads even though there's the slim 321 * possibility of read() fop errors forcing a re-init of the OA buffer 322 * pointers. A race here could result in a false positive !empty status which 323 * is acceptable. 324 */ 325 static bool gen7_oa_buffer_is_empty_fop_unlocked(struct drm_i915_private *dev_priv) 326 { 327 int report_size = dev_priv->perf.oa.oa_buffer.format_size; 328 u32 oastatus2 = I915_READ(GEN7_OASTATUS2); 329 u32 oastatus1 = I915_READ(GEN7_OASTATUS1); 330 u32 head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; 331 u32 tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; 332 333 return OA_TAKEN(tail, head) < 334 dev_priv->perf.oa.tail_margin + report_size; 335 } 336 337 /** 338 * append_oa_status - Appends a status record to a userspace read() buffer. 339 * @stream: An i915-perf stream opened for OA metrics 340 * @buf: destination buffer given by userspace 341 * @count: the number of bytes userspace wants to read 342 * @offset: (inout): the current position for writing into @buf 343 * @type: The kind of status to report to userspace 344 * 345 * Writes a status record (such as `DRM_I915_PERF_RECORD_OA_REPORT_LOST`) 346 * into the userspace read() buffer. 347 * 348 * The @buf @offset will only be updated on success. 349 * 350 * Returns: 0 on success, negative error code on failure. 351 */ 352 static int append_oa_status(struct i915_perf_stream *stream, 353 char __user *buf, 354 size_t count, 355 size_t *offset, 356 enum drm_i915_perf_record_type type) 357 { 358 struct drm_i915_perf_record_header header = { type, 0, sizeof(header) }; 359 360 if ((count - *offset) < header.size) 361 return -ENOSPC; 362 363 if (copy_to_user(buf + *offset, &header, sizeof(header))) 364 return -EFAULT; 365 366 (*offset) += header.size; 367 368 return 0; 369 } 370 371 /** 372 * append_oa_sample - Copies single OA report into userspace read() buffer. 373 * @stream: An i915-perf stream opened for OA metrics 374 * @buf: destination buffer given by userspace 375 * @count: the number of bytes userspace wants to read 376 * @offset: (inout): the current position for writing into @buf 377 * @report: A single OA report to (optionally) include as part of the sample 378 * 379 * The contents of a sample are configured through `DRM_I915_PERF_PROP_SAMPLE_*` 380 * properties when opening a stream, tracked as `stream->sample_flags`. This 381 * function copies the requested components of a single sample to the given 382 * read() @buf. 383 * 384 * The @buf @offset will only be updated on success. 385 * 386 * Returns: 0 on success, negative error code on failure. 387 */ 388 static int append_oa_sample(struct i915_perf_stream *stream, 389 char __user *buf, 390 size_t count, 391 size_t *offset, 392 const u8 *report) 393 { 394 struct drm_i915_private *dev_priv = stream->dev_priv; 395 int report_size = dev_priv->perf.oa.oa_buffer.format_size; 396 struct drm_i915_perf_record_header header; 397 u32 sample_flags = stream->sample_flags; 398 399 header.type = DRM_I915_PERF_RECORD_SAMPLE; 400 header.pad = 0; 401 header.size = stream->sample_size; 402 403 if ((count - *offset) < header.size) 404 return -ENOSPC; 405 406 buf += *offset; 407 if (copy_to_user(buf, &header, sizeof(header))) 408 return -EFAULT; 409 buf += sizeof(header); 410 411 if (sample_flags & SAMPLE_OA_REPORT) { 412 if (copy_to_user(buf, report, report_size)) 413 return -EFAULT; 414 } 415 416 (*offset) += header.size; 417 418 return 0; 419 } 420 421 /** 422 * Copies all buffered OA reports into userspace read() buffer. 423 * @stream: An i915-perf stream opened for OA metrics 424 * @buf: destination buffer given by userspace 425 * @count: the number of bytes userspace wants to read 426 * @offset: (inout): the current position for writing into @buf 427 * @head_ptr: (inout): the current oa buffer cpu read position 428 * @tail: the current oa buffer gpu write position 429 * 430 * Notably any error condition resulting in a short read (-%ENOSPC or 431 * -%EFAULT) will be returned even though one or more records may 432 * have been successfully copied. In this case it's up to the caller 433 * to decide if the error should be squashed before returning to 434 * userspace. 435 * 436 * Note: reports are consumed from the head, and appended to the 437 * tail, so the head chases the tail?... If you think that's mad 438 * and back-to-front you're not alone, but this follows the 439 * Gen PRM naming convention. 440 * 441 * Returns: 0 on success, negative error code on failure. 442 */ 443 static int gen7_append_oa_reports(struct i915_perf_stream *stream, 444 char __user *buf, 445 size_t count, 446 size_t *offset, 447 u32 *head_ptr, 448 u32 tail) 449 { 450 struct drm_i915_private *dev_priv = stream->dev_priv; 451 int report_size = dev_priv->perf.oa.oa_buffer.format_size; 452 u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; 453 int tail_margin = dev_priv->perf.oa.tail_margin; 454 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); 455 u32 mask = (OA_BUFFER_SIZE - 1); 456 u32 head; 457 u32 taken; 458 int ret = 0; 459 460 if (WARN_ON(!stream->enabled)) 461 return -EIO; 462 463 head = *head_ptr - gtt_offset; 464 tail -= gtt_offset; 465 466 /* The OA unit is expected to wrap the tail pointer according to the OA 467 * buffer size and since we should never write a misaligned head 468 * pointer we don't expect to read one back either... 469 */ 470 if (tail > OA_BUFFER_SIZE || head > OA_BUFFER_SIZE || 471 head % report_size) { 472 DRM_ERROR("Inconsistent OA buffer pointer (head = %u, tail = %u): force restart\n", 473 head, tail); 474 dev_priv->perf.oa.ops.oa_disable(dev_priv); 475 dev_priv->perf.oa.ops.oa_enable(dev_priv); 476 *head_ptr = I915_READ(GEN7_OASTATUS2) & 477 GEN7_OASTATUS2_HEAD_MASK; 478 return -EIO; 479 } 480 481 482 /* The tail pointer increases in 64 byte increments, not in report_size 483 * steps... 484 */ 485 tail &= ~(report_size - 1); 486 487 /* Move the tail pointer back by the current tail_margin to account for 488 * the possibility that the latest reports may not have really landed 489 * in memory yet... 490 */ 491 492 if (OA_TAKEN(tail, head) < report_size + tail_margin) 493 return -EAGAIN; 494 495 tail -= tail_margin; 496 tail &= mask; 497 498 for (/* none */; 499 (taken = OA_TAKEN(tail, head)); 500 head = (head + report_size) & mask) { 501 u8 *report = oa_buf_base + head; 502 u32 *report32 = (void *)report; 503 504 /* All the report sizes factor neatly into the buffer 505 * size so we never expect to see a report split 506 * between the beginning and end of the buffer. 507 * 508 * Given the initial alignment check a misalignment 509 * here would imply a driver bug that would result 510 * in an overrun. 511 */ 512 if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { 513 DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); 514 break; 515 } 516 517 /* The report-ID field for periodic samples includes 518 * some undocumented flags related to what triggered 519 * the report and is never expected to be zero so we 520 * can check that the report isn't invalid before 521 * copying it to userspace... 522 */ 523 if (report32[0] == 0) { 524 DRM_NOTE("Skipping spurious, invalid OA report\n"); 525 continue; 526 } 527 528 ret = append_oa_sample(stream, buf, count, offset, report); 529 if (ret) 530 break; 531 532 /* The above report-id field sanity check is based on 533 * the assumption that the OA buffer is initially 534 * zeroed and we reset the field after copying so the 535 * check is still meaningful once old reports start 536 * being overwritten. 537 */ 538 report32[0] = 0; 539 } 540 541 *head_ptr = gtt_offset + head; 542 543 return ret; 544 } 545 546 /** 547 * gen7_oa_read - copy status records then buffered OA reports 548 * @stream: An i915-perf stream opened for OA metrics 549 * @buf: destination buffer given by userspace 550 * @count: the number of bytes userspace wants to read 551 * @offset: (inout): the current position for writing into @buf 552 * 553 * Checks Gen 7 specific OA unit status registers and if necessary appends 554 * corresponding status records for userspace (such as for a buffer full 555 * condition) and then initiate appending any buffered OA reports. 556 * 557 * Updates @offset according to the number of bytes successfully copied into 558 * the userspace buffer. 559 * 560 * Returns: zero on success or a negative error code 561 */ 562 static int gen7_oa_read(struct i915_perf_stream *stream, 563 char __user *buf, 564 size_t count, 565 size_t *offset) 566 { 567 struct drm_i915_private *dev_priv = stream->dev_priv; 568 int report_size = dev_priv->perf.oa.oa_buffer.format_size; 569 u32 oastatus2; 570 u32 oastatus1; 571 u32 head; 572 u32 tail; 573 int ret; 574 575 if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) 576 return -EIO; 577 578 oastatus2 = I915_READ(GEN7_OASTATUS2); 579 oastatus1 = I915_READ(GEN7_OASTATUS1); 580 581 head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; 582 tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; 583 584 /* XXX: On Haswell we don't have a safe way to clear oastatus1 585 * bits while the OA unit is enabled (while the tail pointer 586 * may be updated asynchronously) so we ignore status bits 587 * that have already been reported to userspace. 588 */ 589 oastatus1 &= ~dev_priv->perf.oa.gen7_latched_oastatus1; 590 591 /* We treat OABUFFER_OVERFLOW as a significant error: 592 * 593 * - The status can be interpreted to mean that the buffer is 594 * currently full (with a higher precedence than OA_TAKEN() 595 * which will start to report a near-empty buffer after an 596 * overflow) but it's awkward that we can't clear the status 597 * on Haswell, so without a reset we won't be able to catch 598 * the state again. 599 * 600 * - Since it also implies the HW has started overwriting old 601 * reports it may also affect our sanity checks for invalid 602 * reports when copying to userspace that assume new reports 603 * are being written to cleared memory. 604 * 605 * - In the future we may want to introduce a flight recorder 606 * mode where the driver will automatically maintain a safe 607 * guard band between head/tail, avoiding this overflow 608 * condition, but we avoid the added driver complexity for 609 * now. 610 */ 611 if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) { 612 ret = append_oa_status(stream, buf, count, offset, 613 DRM_I915_PERF_RECORD_OA_BUFFER_LOST); 614 if (ret) 615 return ret; 616 617 DRM_DEBUG("OA buffer overflow: force restart\n"); 618 619 dev_priv->perf.oa.ops.oa_disable(dev_priv); 620 dev_priv->perf.oa.ops.oa_enable(dev_priv); 621 622 oastatus2 = I915_READ(GEN7_OASTATUS2); 623 oastatus1 = I915_READ(GEN7_OASTATUS1); 624 625 head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; 626 tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; 627 } 628 629 if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) { 630 ret = append_oa_status(stream, buf, count, offset, 631 DRM_I915_PERF_RECORD_OA_REPORT_LOST); 632 if (ret) 633 return ret; 634 dev_priv->perf.oa.gen7_latched_oastatus1 |= 635 GEN7_OASTATUS1_REPORT_LOST; 636 } 637 638 ret = gen7_append_oa_reports(stream, buf, count, offset, 639 &head, tail); 640 641 /* All the report sizes are a power of two and the 642 * head should always be incremented by some multiple 643 * of the report size. 644 * 645 * A warning here, but notably if we later read back a 646 * misaligned pointer we will treat that as a bug since 647 * it could lead to a buffer overrun. 648 */ 649 WARN_ONCE(head & (report_size - 1), 650 "i915: Writing misaligned OA head pointer"); 651 652 /* Note: we update the head pointer here even if an error 653 * was returned since the error may represent a short read 654 * where some some reports were successfully copied. 655 */ 656 I915_WRITE(GEN7_OASTATUS2, 657 ((head & GEN7_OASTATUS2_HEAD_MASK) | 658 OA_MEM_SELECT_GGTT)); 659 660 return ret; 661 } 662 663 /** 664 * i915_oa_wait_unlocked - handles blocking IO until OA data available 665 * @stream: An i915-perf stream opened for OA metrics 666 * 667 * Called when userspace tries to read() from a blocking stream FD opened 668 * for OA metrics. It waits until the hrtimer callback finds a non-empty 669 * OA buffer and wakes us. 670 * 671 * Note: it's acceptable to have this return with some false positives 672 * since any subsequent read handling will return -EAGAIN if there isn't 673 * really data ready for userspace yet. 674 * 675 * Returns: zero on success or a negative error code 676 */ 677 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) 678 { 679 struct drm_i915_private *dev_priv = stream->dev_priv; 680 681 /* We would wait indefinitely if periodic sampling is not enabled */ 682 if (!dev_priv->perf.oa.periodic) 683 return -EIO; 684 685 /* Note: the oa_buffer_is_empty() condition is ok to run unlocked as it 686 * just performs mmio reads of the OA buffer head + tail pointers and 687 * it's assumed we're handling some operation that implies the stream 688 * can't be destroyed until completion (such as a read()) that ensures 689 * the device + OA buffer can't disappear 690 */ 691 return wait_event_interruptible(dev_priv->perf.oa.poll_wq, 692 !dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)); 693 } 694 695 /** 696 * i915_oa_poll_wait - call poll_wait() for an OA stream poll() 697 * @stream: An i915-perf stream opened for OA metrics 698 * @file: An i915 perf stream file 699 * @wait: poll() state table 700 * 701 * For handling userspace polling on an i915 perf stream opened for OA metrics, 702 * this starts a poll_wait with the wait queue that our hrtimer callback wakes 703 * when it sees data ready to read in the circular OA buffer. 704 */ 705 static void i915_oa_poll_wait(struct i915_perf_stream *stream, 706 struct file *file, 707 poll_table *wait) 708 { 709 struct drm_i915_private *dev_priv = stream->dev_priv; 710 711 poll_wait(file, &dev_priv->perf.oa.poll_wq, wait); 712 } 713 714 /** 715 * i915_oa_read - just calls through to &i915_oa_ops->read 716 * @stream: An i915-perf stream opened for OA metrics 717 * @buf: destination buffer given by userspace 718 * @count: the number of bytes userspace wants to read 719 * @offset: (inout): the current position for writing into @buf 720 * 721 * Updates @offset according to the number of bytes successfully copied into 722 * the userspace buffer. 723 * 724 * Returns: zero on success or a negative error code 725 */ 726 static int i915_oa_read(struct i915_perf_stream *stream, 727 char __user *buf, 728 size_t count, 729 size_t *offset) 730 { 731 struct drm_i915_private *dev_priv = stream->dev_priv; 732 733 return dev_priv->perf.oa.ops.read(stream, buf, count, offset); 734 } 735 736 /** 737 * oa_get_render_ctx_id - determine and hold ctx hw id 738 * @stream: An i915-perf stream opened for OA metrics 739 * 740 * Determine the render context hw id, and ensure it remains fixed for the 741 * lifetime of the stream. This ensures that we don't have to worry about 742 * updating the context ID in OACONTROL on the fly. 743 * 744 * Returns: zero on success or a negative error code 745 */ 746 static int oa_get_render_ctx_id(struct i915_perf_stream *stream) 747 { 748 struct drm_i915_private *dev_priv = stream->dev_priv; 749 struct intel_engine_cs *engine = dev_priv->engine[RCS]; 750 int ret; 751 752 ret = i915_mutex_lock_interruptible(&dev_priv->drm); 753 if (ret) 754 return ret; 755 756 /* As the ID is the gtt offset of the context's vma we pin 757 * the vma to ensure the ID remains fixed. 758 * 759 * NB: implied RCS engine... 760 */ 761 ret = engine->context_pin(engine, stream->ctx); 762 if (ret) 763 goto unlock; 764 765 /* Explicitly track the ID (instead of calling i915_ggtt_offset() 766 * on the fly) considering the difference with gen8+ and 767 * execlists 768 */ 769 dev_priv->perf.oa.specific_ctx_id = 770 i915_ggtt_offset(stream->ctx->engine[engine->id].state); 771 772 unlock: 773 mutex_unlock(&dev_priv->drm.struct_mutex); 774 775 return ret; 776 } 777 778 /** 779 * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold 780 * @stream: An i915-perf stream opened for OA metrics 781 * 782 * In case anything needed doing to ensure the context HW ID would remain valid 783 * for the lifetime of the stream, then that can be undone here. 784 */ 785 static void oa_put_render_ctx_id(struct i915_perf_stream *stream) 786 { 787 struct drm_i915_private *dev_priv = stream->dev_priv; 788 struct intel_engine_cs *engine = dev_priv->engine[RCS]; 789 790 mutex_lock(&dev_priv->drm.struct_mutex); 791 792 dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID; 793 engine->context_unpin(engine, stream->ctx); 794 795 mutex_unlock(&dev_priv->drm.struct_mutex); 796 } 797 798 static void 799 free_oa_buffer(struct drm_i915_private *i915) 800 { 801 mutex_lock(&i915->drm.struct_mutex); 802 803 i915_gem_object_unpin_map(i915->perf.oa.oa_buffer.vma->obj); 804 i915_vma_unpin(i915->perf.oa.oa_buffer.vma); 805 i915_gem_object_put(i915->perf.oa.oa_buffer.vma->obj); 806 807 i915->perf.oa.oa_buffer.vma = NULL; 808 i915->perf.oa.oa_buffer.vaddr = NULL; 809 810 mutex_unlock(&i915->drm.struct_mutex); 811 } 812 813 static void i915_oa_stream_destroy(struct i915_perf_stream *stream) 814 { 815 struct drm_i915_private *dev_priv = stream->dev_priv; 816 817 BUG_ON(stream != dev_priv->perf.oa.exclusive_stream); 818 819 dev_priv->perf.oa.ops.disable_metric_set(dev_priv); 820 821 free_oa_buffer(dev_priv); 822 823 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 824 intel_runtime_pm_put(dev_priv); 825 826 if (stream->ctx) 827 oa_put_render_ctx_id(stream); 828 829 dev_priv->perf.oa.exclusive_stream = NULL; 830 } 831 832 static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv) 833 { 834 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); 835 836 /* Pre-DevBDW: OABUFFER must be set with counters off, 837 * before OASTATUS1, but after OASTATUS2 838 */ 839 I915_WRITE(GEN7_OASTATUS2, gtt_offset | OA_MEM_SELECT_GGTT); /* head */ 840 I915_WRITE(GEN7_OABUFFER, gtt_offset); 841 I915_WRITE(GEN7_OASTATUS1, gtt_offset | OABUFFER_SIZE_16M); /* tail */ 842 843 /* On Haswell we have to track which OASTATUS1 flags we've 844 * already seen since they can't be cleared while periodic 845 * sampling is enabled. 846 */ 847 dev_priv->perf.oa.gen7_latched_oastatus1 = 0; 848 849 /* NB: although the OA buffer will initially be allocated 850 * zeroed via shmfs (and so this memset is redundant when 851 * first allocating), we may re-init the OA buffer, either 852 * when re-enabling a stream or in error/reset paths. 853 * 854 * The reason we clear the buffer for each re-init is for the 855 * sanity check in gen7_append_oa_reports() that looks at the 856 * report-id field to make sure it's non-zero which relies on 857 * the assumption that new reports are being written to zeroed 858 * memory... 859 */ 860 memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); 861 862 /* Maybe make ->pollin per-stream state if we support multiple 863 * concurrent streams in the future. 864 */ 865 dev_priv->perf.oa.pollin = false; 866 } 867 868 static int alloc_oa_buffer(struct drm_i915_private *dev_priv) 869 { 870 struct drm_i915_gem_object *bo; 871 struct i915_vma *vma; 872 int ret; 873 874 if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma)) 875 return -ENODEV; 876 877 ret = i915_mutex_lock_interruptible(&dev_priv->drm); 878 if (ret) 879 return ret; 880 881 BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE); 882 BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M); 883 884 bo = i915_gem_object_create(dev_priv, OA_BUFFER_SIZE); 885 if (IS_ERR(bo)) { 886 DRM_ERROR("Failed to allocate OA buffer\n"); 887 ret = PTR_ERR(bo); 888 goto unlock; 889 } 890 891 ret = i915_gem_object_set_cache_level(bo, I915_CACHE_LLC); 892 if (ret) 893 goto err_unref; 894 895 /* PreHSW required 512K alignment, HSW requires 16M */ 896 vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0); 897 if (IS_ERR(vma)) { 898 ret = PTR_ERR(vma); 899 goto err_unref; 900 } 901 dev_priv->perf.oa.oa_buffer.vma = vma; 902 903 dev_priv->perf.oa.oa_buffer.vaddr = 904 i915_gem_object_pin_map(bo, I915_MAP_WB); 905 if (IS_ERR(dev_priv->perf.oa.oa_buffer.vaddr)) { 906 ret = PTR_ERR(dev_priv->perf.oa.oa_buffer.vaddr); 907 goto err_unpin; 908 } 909 910 dev_priv->perf.oa.ops.init_oa_buffer(dev_priv); 911 912 DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n", 913 i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma), 914 dev_priv->perf.oa.oa_buffer.vaddr); 915 916 goto unlock; 917 918 err_unpin: 919 __i915_vma_unpin(vma); 920 921 err_unref: 922 i915_gem_object_put(bo); 923 924 dev_priv->perf.oa.oa_buffer.vaddr = NULL; 925 dev_priv->perf.oa.oa_buffer.vma = NULL; 926 927 unlock: 928 mutex_unlock(&dev_priv->drm.struct_mutex); 929 return ret; 930 } 931 932 static void config_oa_regs(struct drm_i915_private *dev_priv, 933 const struct i915_oa_reg *regs, 934 int n_regs) 935 { 936 int i; 937 938 for (i = 0; i < n_regs; i++) { 939 const struct i915_oa_reg *reg = regs + i; 940 941 I915_WRITE(reg->addr, reg->value); 942 } 943 } 944 945 static int hsw_enable_metric_set(struct drm_i915_private *dev_priv) 946 { 947 int ret = i915_oa_select_metric_set_hsw(dev_priv); 948 949 if (ret) 950 return ret; 951 952 I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) | 953 GT_NOA_ENABLE)); 954 955 /* PRM: 956 * 957 * OA unit is using “crclk” for its functionality. When trunk 958 * level clock gating takes place, OA clock would be gated, 959 * unable to count the events from non-render clock domain. 960 * Render clock gating must be disabled when OA is enabled to 961 * count the events from non-render domain. Unit level clock 962 * gating for RCS should also be disabled. 963 */ 964 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & 965 ~GEN7_DOP_CLOCK_GATE_ENABLE)); 966 I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) | 967 GEN6_CSUNIT_CLOCK_GATE_DISABLE)); 968 969 config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs, 970 dev_priv->perf.oa.mux_regs_len); 971 972 /* It apparently takes a fairly long time for a new MUX 973 * configuration to be be applied after these register writes. 974 * This delay duration was derived empirically based on the 975 * render_basic config but hopefully it covers the maximum 976 * configuration latency. 977 * 978 * As a fallback, the checks in _append_oa_reports() to skip 979 * invalid OA reports do also seem to work to discard reports 980 * generated before this config has completed - albeit not 981 * silently. 982 * 983 * Unfortunately this is essentially a magic number, since we 984 * don't currently know of a reliable mechanism for predicting 985 * how long the MUX config will take to apply and besides 986 * seeing invalid reports we don't know of a reliable way to 987 * explicitly check that the MUX config has landed. 988 * 989 * It's even possible we've miss characterized the underlying 990 * problem - it just seems like the simplest explanation why 991 * a delay at this location would mitigate any invalid reports. 992 */ 993 usleep_range(15000, 20000); 994 995 config_oa_regs(dev_priv, dev_priv->perf.oa.b_counter_regs, 996 dev_priv->perf.oa.b_counter_regs_len); 997 998 return 0; 999 } 1000 1001 static void hsw_disable_metric_set(struct drm_i915_private *dev_priv) 1002 { 1003 I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) & 1004 ~GEN6_CSUNIT_CLOCK_GATE_DISABLE)); 1005 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) | 1006 GEN7_DOP_CLOCK_GATE_ENABLE)); 1007 1008 I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & 1009 ~GT_NOA_ENABLE)); 1010 } 1011 1012 static void gen7_update_oacontrol_locked(struct drm_i915_private *dev_priv) 1013 { 1014 lockdep_assert_held(&dev_priv->perf.hook_lock); 1015 1016 if (dev_priv->perf.oa.exclusive_stream->enabled) { 1017 struct i915_gem_context *ctx = 1018 dev_priv->perf.oa.exclusive_stream->ctx; 1019 u32 ctx_id = dev_priv->perf.oa.specific_ctx_id; 1020 1021 bool periodic = dev_priv->perf.oa.periodic; 1022 u32 period_exponent = dev_priv->perf.oa.period_exponent; 1023 u32 report_format = dev_priv->perf.oa.oa_buffer.format; 1024 1025 I915_WRITE(GEN7_OACONTROL, 1026 (ctx_id & GEN7_OACONTROL_CTX_MASK) | 1027 (period_exponent << 1028 GEN7_OACONTROL_TIMER_PERIOD_SHIFT) | 1029 (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) | 1030 (report_format << GEN7_OACONTROL_FORMAT_SHIFT) | 1031 (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) | 1032 GEN7_OACONTROL_ENABLE); 1033 } else 1034 I915_WRITE(GEN7_OACONTROL, 0); 1035 } 1036 1037 static void gen7_oa_enable(struct drm_i915_private *dev_priv) 1038 { 1039 unsigned long flags; 1040 1041 /* Reset buf pointers so we don't forward reports from before now. 1042 * 1043 * Think carefully if considering trying to avoid this, since it 1044 * also ensures status flags and the buffer itself are cleared 1045 * in error paths, and we have checks for invalid reports based 1046 * on the assumption that certain fields are written to zeroed 1047 * memory which this helps maintains. 1048 */ 1049 gen7_init_oa_buffer(dev_priv); 1050 1051 spin_lock_irqsave(&dev_priv->perf.hook_lock, flags); 1052 gen7_update_oacontrol_locked(dev_priv); 1053 spin_unlock_irqrestore(&dev_priv->perf.hook_lock, flags); 1054 } 1055 1056 /** 1057 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream 1058 * @stream: An i915 perf stream opened for OA metrics 1059 * 1060 * [Re]enables hardware periodic sampling according to the period configured 1061 * when opening the stream. This also starts a hrtimer that will periodically 1062 * check for data in the circular OA buffer for notifying userspace (e.g. 1063 * during a read() or poll()). 1064 */ 1065 static void i915_oa_stream_enable(struct i915_perf_stream *stream) 1066 { 1067 struct drm_i915_private *dev_priv = stream->dev_priv; 1068 1069 dev_priv->perf.oa.ops.oa_enable(dev_priv); 1070 1071 if (dev_priv->perf.oa.periodic) 1072 hrtimer_start(&dev_priv->perf.oa.poll_check_timer, 1073 ns_to_ktime(POLL_PERIOD), 1074 HRTIMER_MODE_REL_PINNED); 1075 } 1076 1077 static void gen7_oa_disable(struct drm_i915_private *dev_priv) 1078 { 1079 I915_WRITE(GEN7_OACONTROL, 0); 1080 } 1081 1082 /** 1083 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream 1084 * @stream: An i915 perf stream opened for OA metrics 1085 * 1086 * Stops the OA unit from periodically writing counter reports into the 1087 * circular OA buffer. This also stops the hrtimer that periodically checks for 1088 * data in the circular OA buffer, for notifying userspace. 1089 */ 1090 static void i915_oa_stream_disable(struct i915_perf_stream *stream) 1091 { 1092 struct drm_i915_private *dev_priv = stream->dev_priv; 1093 1094 dev_priv->perf.oa.ops.oa_disable(dev_priv); 1095 1096 if (dev_priv->perf.oa.periodic) 1097 hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer); 1098 } 1099 1100 static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent) 1101 { 1102 return div_u64(1000000000ULL * (2ULL << exponent), 1103 dev_priv->perf.oa.timestamp_frequency); 1104 } 1105 1106 static const struct i915_perf_stream_ops i915_oa_stream_ops = { 1107 .destroy = i915_oa_stream_destroy, 1108 .enable = i915_oa_stream_enable, 1109 .disable = i915_oa_stream_disable, 1110 .wait_unlocked = i915_oa_wait_unlocked, 1111 .poll_wait = i915_oa_poll_wait, 1112 .read = i915_oa_read, 1113 }; 1114 1115 /** 1116 * i915_oa_stream_init - validate combined props for OA stream and init 1117 * @stream: An i915 perf stream 1118 * @param: The open parameters passed to `DRM_I915_PERF_OPEN` 1119 * @props: The property state that configures stream (individually validated) 1120 * 1121 * While read_properties_unlocked() validates properties in isolation it 1122 * doesn't ensure that the combination necessarily makes sense. 1123 * 1124 * At this point it has been determined that userspace wants a stream of 1125 * OA metrics, but still we need to further validate the combined 1126 * properties are OK. 1127 * 1128 * If the configuration makes sense then we can allocate memory for 1129 * a circular OA buffer and apply the requested metric set configuration. 1130 * 1131 * Returns: zero on success or a negative error code. 1132 */ 1133 static int i915_oa_stream_init(struct i915_perf_stream *stream, 1134 struct drm_i915_perf_open_param *param, 1135 struct perf_open_properties *props) 1136 { 1137 struct drm_i915_private *dev_priv = stream->dev_priv; 1138 int format_size; 1139 int ret; 1140 1141 /* If the sysfs metrics/ directory wasn't registered for some 1142 * reason then don't let userspace try their luck with config 1143 * IDs 1144 */ 1145 if (!dev_priv->perf.metrics_kobj) { 1146 DRM_DEBUG("OA metrics weren't advertised via sysfs\n"); 1147 return -EINVAL; 1148 } 1149 1150 if (!(props->sample_flags & SAMPLE_OA_REPORT)) { 1151 DRM_DEBUG("Only OA report sampling supported\n"); 1152 return -EINVAL; 1153 } 1154 1155 if (!dev_priv->perf.oa.ops.init_oa_buffer) { 1156 DRM_DEBUG("OA unit not supported\n"); 1157 return -ENODEV; 1158 } 1159 1160 /* To avoid the complexity of having to accurately filter 1161 * counter reports and marshal to the appropriate client 1162 * we currently only allow exclusive access 1163 */ 1164 if (dev_priv->perf.oa.exclusive_stream) { 1165 DRM_DEBUG("OA unit already in use\n"); 1166 return -EBUSY; 1167 } 1168 1169 if (!props->metrics_set) { 1170 DRM_DEBUG("OA metric set not specified\n"); 1171 return -EINVAL; 1172 } 1173 1174 if (!props->oa_format) { 1175 DRM_DEBUG("OA report format not specified\n"); 1176 return -EINVAL; 1177 } 1178 1179 stream->sample_size = sizeof(struct drm_i915_perf_record_header); 1180 1181 format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size; 1182 1183 stream->sample_flags |= SAMPLE_OA_REPORT; 1184 stream->sample_size += format_size; 1185 1186 dev_priv->perf.oa.oa_buffer.format_size = format_size; 1187 if (WARN_ON(dev_priv->perf.oa.oa_buffer.format_size == 0)) 1188 return -EINVAL; 1189 1190 dev_priv->perf.oa.oa_buffer.format = 1191 dev_priv->perf.oa.oa_formats[props->oa_format].format; 1192 1193 dev_priv->perf.oa.metrics_set = props->metrics_set; 1194 1195 dev_priv->perf.oa.periodic = props->oa_periodic; 1196 if (dev_priv->perf.oa.periodic) { 1197 u32 tail; 1198 1199 dev_priv->perf.oa.period_exponent = props->oa_period_exponent; 1200 1201 /* See comment for OA_TAIL_MARGIN_NSEC for details 1202 * about this tail_margin... 1203 */ 1204 tail = div64_u64(OA_TAIL_MARGIN_NSEC, 1205 oa_exponent_to_ns(dev_priv, 1206 props->oa_period_exponent)); 1207 dev_priv->perf.oa.tail_margin = (tail + 1) * format_size; 1208 } 1209 1210 if (stream->ctx) { 1211 ret = oa_get_render_ctx_id(stream); 1212 if (ret) 1213 return ret; 1214 } 1215 1216 /* PRM - observability performance counters: 1217 * 1218 * OACONTROL, performance counter enable, note: 1219 * 1220 * "When this bit is set, in order to have coherent counts, 1221 * RC6 power state and trunk clock gating must be disabled. 1222 * This can be achieved by programming MMIO registers as 1223 * 0xA094=0 and 0xA090[31]=1" 1224 * 1225 * In our case we are expecting that taking pm + FORCEWAKE 1226 * references will effectively disable RC6. 1227 */ 1228 intel_runtime_pm_get(dev_priv); 1229 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 1230 1231 ret = alloc_oa_buffer(dev_priv); 1232 if (ret) 1233 goto err_oa_buf_alloc; 1234 1235 ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv); 1236 if (ret) 1237 goto err_enable; 1238 1239 stream->ops = &i915_oa_stream_ops; 1240 1241 dev_priv->perf.oa.exclusive_stream = stream; 1242 1243 return 0; 1244 1245 err_enable: 1246 free_oa_buffer(dev_priv); 1247 1248 err_oa_buf_alloc: 1249 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 1250 intel_runtime_pm_put(dev_priv); 1251 if (stream->ctx) 1252 oa_put_render_ctx_id(stream); 1253 1254 return ret; 1255 } 1256 1257 /** 1258 * i915_perf_read_locked - &i915_perf_stream_ops->read with error normalisation 1259 * @stream: An i915 perf stream 1260 * @file: An i915 perf stream file 1261 * @buf: destination buffer given by userspace 1262 * @count: the number of bytes userspace wants to read 1263 * @ppos: (inout) file seek position (unused) 1264 * 1265 * Besides wrapping &i915_perf_stream_ops->read this provides a common place to 1266 * ensure that if we've successfully copied any data then reporting that takes 1267 * precedence over any internal error status, so the data isn't lost. 1268 * 1269 * For example ret will be -ENOSPC whenever there is more buffered data than 1270 * can be copied to userspace, but that's only interesting if we weren't able 1271 * to copy some data because it implies the userspace buffer is too small to 1272 * receive a single record (and we never split records). 1273 * 1274 * Another case with ret == -EFAULT is more of a grey area since it would seem 1275 * like bad form for userspace to ask us to overrun its buffer, but the user 1276 * knows best: 1277 * 1278 * http://yarchive.net/comp/linux/partial_reads_writes.html 1279 * 1280 * Returns: The number of bytes copied or a negative error code on failure. 1281 */ 1282 static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream, 1283 struct file *file, 1284 char __user *buf, 1285 size_t count, 1286 loff_t *ppos) 1287 { 1288 /* Note we keep the offset (aka bytes read) separate from any 1289 * error status so that the final check for whether we return 1290 * the bytes read with a higher precedence than any error (see 1291 * comment below) doesn't need to be handled/duplicated in 1292 * stream->ops->read() implementations. 1293 */ 1294 size_t offset = 0; 1295 int ret = stream->ops->read(stream, buf, count, &offset); 1296 1297 return offset ?: (ret ?: -EAGAIN); 1298 } 1299 1300 /** 1301 * i915_perf_read - handles read() FOP for i915 perf stream FDs 1302 * @file: An i915 perf stream file 1303 * @buf: destination buffer given by userspace 1304 * @count: the number of bytes userspace wants to read 1305 * @ppos: (inout) file seek position (unused) 1306 * 1307 * The entry point for handling a read() on a stream file descriptor from 1308 * userspace. Most of the work is left to the i915_perf_read_locked() and 1309 * &i915_perf_stream_ops->read but to save having stream implementations (of 1310 * which we might have multiple later) we handle blocking read here. 1311 * 1312 * We can also consistently treat trying to read from a disabled stream 1313 * as an IO error so implementations can assume the stream is enabled 1314 * while reading. 1315 * 1316 * Returns: The number of bytes copied or a negative error code on failure. 1317 */ 1318 static ssize_t i915_perf_read(struct file *file, 1319 char __user *buf, 1320 size_t count, 1321 loff_t *ppos) 1322 { 1323 struct i915_perf_stream *stream = file->private_data; 1324 struct drm_i915_private *dev_priv = stream->dev_priv; 1325 ssize_t ret; 1326 1327 /* To ensure it's handled consistently we simply treat all reads of a 1328 * disabled stream as an error. In particular it might otherwise lead 1329 * to a deadlock for blocking file descriptors... 1330 */ 1331 if (!stream->enabled) 1332 return -EIO; 1333 1334 if (!(file->f_flags & O_NONBLOCK)) { 1335 /* There's the small chance of false positives from 1336 * stream->ops->wait_unlocked. 1337 * 1338 * E.g. with single context filtering since we only wait until 1339 * oabuffer has >= 1 report we don't immediately know whether 1340 * any reports really belong to the current context 1341 */ 1342 do { 1343 ret = stream->ops->wait_unlocked(stream); 1344 if (ret) 1345 return ret; 1346 1347 mutex_lock(&dev_priv->perf.lock); 1348 ret = i915_perf_read_locked(stream, file, 1349 buf, count, ppos); 1350 mutex_unlock(&dev_priv->perf.lock); 1351 } while (ret == -EAGAIN); 1352 } else { 1353 mutex_lock(&dev_priv->perf.lock); 1354 ret = i915_perf_read_locked(stream, file, buf, count, ppos); 1355 mutex_unlock(&dev_priv->perf.lock); 1356 } 1357 1358 if (ret >= 0) { 1359 /* Maybe make ->pollin per-stream state if we support multiple 1360 * concurrent streams in the future. 1361 */ 1362 dev_priv->perf.oa.pollin = false; 1363 } 1364 1365 return ret; 1366 } 1367 1368 static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer) 1369 { 1370 struct drm_i915_private *dev_priv = 1371 container_of(hrtimer, typeof(*dev_priv), 1372 perf.oa.poll_check_timer); 1373 1374 if (!dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)) { 1375 dev_priv->perf.oa.pollin = true; 1376 wake_up(&dev_priv->perf.oa.poll_wq); 1377 } 1378 1379 hrtimer_forward_now(hrtimer, ns_to_ktime(POLL_PERIOD)); 1380 1381 return HRTIMER_RESTART; 1382 } 1383 1384 /** 1385 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream 1386 * @dev_priv: i915 device instance 1387 * @stream: An i915 perf stream 1388 * @file: An i915 perf stream file 1389 * @wait: poll() state table 1390 * 1391 * For handling userspace polling on an i915 perf stream, this calls through to 1392 * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that 1393 * will be woken for new stream data. 1394 * 1395 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize 1396 * with any non-file-operation driver hooks. 1397 * 1398 * Returns: any poll events that are ready without sleeping 1399 */ 1400 static unsigned int i915_perf_poll_locked(struct drm_i915_private *dev_priv, 1401 struct i915_perf_stream *stream, 1402 struct file *file, 1403 poll_table *wait) 1404 { 1405 unsigned int events = 0; 1406 1407 stream->ops->poll_wait(stream, file, wait); 1408 1409 /* Note: we don't explicitly check whether there's something to read 1410 * here since this path may be very hot depending on what else 1411 * userspace is polling, or on the timeout in use. We rely solely on 1412 * the hrtimer/oa_poll_check_timer_cb to notify us when there are 1413 * samples to read. 1414 */ 1415 if (dev_priv->perf.oa.pollin) 1416 events |= POLLIN; 1417 1418 return events; 1419 } 1420 1421 /** 1422 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream 1423 * @file: An i915 perf stream file 1424 * @wait: poll() state table 1425 * 1426 * For handling userspace polling on an i915 perf stream, this ensures 1427 * poll_wait() gets called with a wait queue that will be woken for new stream 1428 * data. 1429 * 1430 * Note: Implementation deferred to i915_perf_poll_locked() 1431 * 1432 * Returns: any poll events that are ready without sleeping 1433 */ 1434 static unsigned int i915_perf_poll(struct file *file, poll_table *wait) 1435 { 1436 struct i915_perf_stream *stream = file->private_data; 1437 struct drm_i915_private *dev_priv = stream->dev_priv; 1438 int ret; 1439 1440 mutex_lock(&dev_priv->perf.lock); 1441 ret = i915_perf_poll_locked(dev_priv, stream, file, wait); 1442 mutex_unlock(&dev_priv->perf.lock); 1443 1444 return ret; 1445 } 1446 1447 /** 1448 * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl 1449 * @stream: A disabled i915 perf stream 1450 * 1451 * [Re]enables the associated capture of data for this stream. 1452 * 1453 * If a stream was previously enabled then there's currently no intention 1454 * to provide userspace any guarantee about the preservation of previously 1455 * buffered data. 1456 */ 1457 static void i915_perf_enable_locked(struct i915_perf_stream *stream) 1458 { 1459 if (stream->enabled) 1460 return; 1461 1462 /* Allow stream->ops->enable() to refer to this */ 1463 stream->enabled = true; 1464 1465 if (stream->ops->enable) 1466 stream->ops->enable(stream); 1467 } 1468 1469 /** 1470 * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl 1471 * @stream: An enabled i915 perf stream 1472 * 1473 * Disables the associated capture of data for this stream. 1474 * 1475 * The intention is that disabling an re-enabling a stream will ideally be 1476 * cheaper than destroying and re-opening a stream with the same configuration, 1477 * though there are no formal guarantees about what state or buffered data 1478 * must be retained between disabling and re-enabling a stream. 1479 * 1480 * Note: while a stream is disabled it's considered an error for userspace 1481 * to attempt to read from the stream (-EIO). 1482 */ 1483 static void i915_perf_disable_locked(struct i915_perf_stream *stream) 1484 { 1485 if (!stream->enabled) 1486 return; 1487 1488 /* Allow stream->ops->disable() to refer to this */ 1489 stream->enabled = false; 1490 1491 if (stream->ops->disable) 1492 stream->ops->disable(stream); 1493 } 1494 1495 /** 1496 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs 1497 * @stream: An i915 perf stream 1498 * @cmd: the ioctl request 1499 * @arg: the ioctl data 1500 * 1501 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize 1502 * with any non-file-operation driver hooks. 1503 * 1504 * Returns: zero on success or a negative error code. Returns -EINVAL for 1505 * an unknown ioctl request. 1506 */ 1507 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, 1508 unsigned int cmd, 1509 unsigned long arg) 1510 { 1511 switch (cmd) { 1512 case I915_PERF_IOCTL_ENABLE: 1513 i915_perf_enable_locked(stream); 1514 return 0; 1515 case I915_PERF_IOCTL_DISABLE: 1516 i915_perf_disable_locked(stream); 1517 return 0; 1518 } 1519 1520 return -EINVAL; 1521 } 1522 1523 /** 1524 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs 1525 * @file: An i915 perf stream file 1526 * @cmd: the ioctl request 1527 * @arg: the ioctl data 1528 * 1529 * Implementation deferred to i915_perf_ioctl_locked(). 1530 * 1531 * Returns: zero on success or a negative error code. Returns -EINVAL for 1532 * an unknown ioctl request. 1533 */ 1534 static long i915_perf_ioctl(struct file *file, 1535 unsigned int cmd, 1536 unsigned long arg) 1537 { 1538 struct i915_perf_stream *stream = file->private_data; 1539 struct drm_i915_private *dev_priv = stream->dev_priv; 1540 long ret; 1541 1542 mutex_lock(&dev_priv->perf.lock); 1543 ret = i915_perf_ioctl_locked(stream, cmd, arg); 1544 mutex_unlock(&dev_priv->perf.lock); 1545 1546 return ret; 1547 } 1548 1549 /** 1550 * i915_perf_destroy_locked - destroy an i915 perf stream 1551 * @stream: An i915 perf stream 1552 * 1553 * Frees all resources associated with the given i915 perf @stream, disabling 1554 * any associated data capture in the process. 1555 * 1556 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize 1557 * with any non-file-operation driver hooks. 1558 */ 1559 static void i915_perf_destroy_locked(struct i915_perf_stream *stream) 1560 { 1561 if (stream->enabled) 1562 i915_perf_disable_locked(stream); 1563 1564 if (stream->ops->destroy) 1565 stream->ops->destroy(stream); 1566 1567 list_del(&stream->link); 1568 1569 if (stream->ctx) 1570 i915_gem_context_put_unlocked(stream->ctx); 1571 1572 kfree(stream); 1573 } 1574 1575 /** 1576 * i915_perf_release - handles userspace close() of a stream file 1577 * @inode: anonymous inode associated with file 1578 * @file: An i915 perf stream file 1579 * 1580 * Cleans up any resources associated with an open i915 perf stream file. 1581 * 1582 * NB: close() can't really fail from the userspace point of view. 1583 * 1584 * Returns: zero on success or a negative error code. 1585 */ 1586 static int i915_perf_release(struct inode *inode, struct file *file) 1587 { 1588 struct i915_perf_stream *stream = file->private_data; 1589 struct drm_i915_private *dev_priv = stream->dev_priv; 1590 1591 mutex_lock(&dev_priv->perf.lock); 1592 i915_perf_destroy_locked(stream); 1593 mutex_unlock(&dev_priv->perf.lock); 1594 1595 return 0; 1596 } 1597 1598 1599 static const struct file_operations fops = { 1600 .owner = THIS_MODULE, 1601 .llseek = no_llseek, 1602 .release = i915_perf_release, 1603 .poll = i915_perf_poll, 1604 .read = i915_perf_read, 1605 .unlocked_ioctl = i915_perf_ioctl, 1606 }; 1607 1608 1609 static struct i915_gem_context * 1610 lookup_context(struct drm_i915_private *dev_priv, 1611 struct drm_i915_file_private *file_priv, 1612 u32 ctx_user_handle) 1613 { 1614 struct i915_gem_context *ctx; 1615 int ret; 1616 1617 ret = i915_mutex_lock_interruptible(&dev_priv->drm); 1618 if (ret) 1619 return ERR_PTR(ret); 1620 1621 ctx = i915_gem_context_lookup(file_priv, ctx_user_handle); 1622 if (!IS_ERR(ctx)) 1623 i915_gem_context_get(ctx); 1624 1625 mutex_unlock(&dev_priv->drm.struct_mutex); 1626 1627 return ctx; 1628 } 1629 1630 /** 1631 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD 1632 * @dev_priv: i915 device instance 1633 * @param: The open parameters passed to 'DRM_I915_PERF_OPEN` 1634 * @props: individually validated u64 property value pairs 1635 * @file: drm file 1636 * 1637 * See i915_perf_ioctl_open() for interface details. 1638 * 1639 * Implements further stream config validation and stream initialization on 1640 * behalf of i915_perf_open_ioctl() with the &drm_i915_private->perf.lock mutex 1641 * taken to serialize with any non-file-operation driver hooks. 1642 * 1643 * Note: at this point the @props have only been validated in isolation and 1644 * it's still necessary to validate that the combination of properties makes 1645 * sense. 1646 * 1647 * In the case where userspace is interested in OA unit metrics then further 1648 * config validation and stream initialization details will be handled by 1649 * i915_oa_stream_init(). The code here should only validate config state that 1650 * will be relevant to all stream types / backends. 1651 * 1652 * Returns: zero on success or a negative error code. 1653 */ 1654 static int 1655 i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv, 1656 struct drm_i915_perf_open_param *param, 1657 struct perf_open_properties *props, 1658 struct drm_file *file) 1659 { 1660 struct i915_gem_context *specific_ctx = NULL; 1661 struct i915_perf_stream *stream = NULL; 1662 unsigned long f_flags = 0; 1663 int stream_fd; 1664 int ret; 1665 1666 if (props->single_context) { 1667 u32 ctx_handle = props->ctx_handle; 1668 struct drm_i915_file_private *file_priv = file->driver_priv; 1669 1670 specific_ctx = lookup_context(dev_priv, file_priv, ctx_handle); 1671 if (IS_ERR(specific_ctx)) { 1672 ret = PTR_ERR(specific_ctx); 1673 if (ret != -EINTR) 1674 DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n", 1675 ctx_handle); 1676 goto err; 1677 } 1678 } 1679 1680 /* Similar to perf's kernel.perf_paranoid_cpu sysctl option 1681 * we check a dev.i915.perf_stream_paranoid sysctl option 1682 * to determine if it's ok to access system wide OA counters 1683 * without CAP_SYS_ADMIN privileges. 1684 */ 1685 if (!specific_ctx && 1686 i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) { 1687 DRM_DEBUG("Insufficient privileges to open system-wide i915 perf stream\n"); 1688 ret = -EACCES; 1689 goto err_ctx; 1690 } 1691 1692 stream = kzalloc(sizeof(*stream), GFP_KERNEL); 1693 if (!stream) { 1694 ret = -ENOMEM; 1695 goto err_ctx; 1696 } 1697 1698 stream->dev_priv = dev_priv; 1699 stream->ctx = specific_ctx; 1700 1701 ret = i915_oa_stream_init(stream, param, props); 1702 if (ret) 1703 goto err_alloc; 1704 1705 /* we avoid simply assigning stream->sample_flags = props->sample_flags 1706 * to have _stream_init check the combination of sample flags more 1707 * thoroughly, but still this is the expected result at this point. 1708 */ 1709 if (WARN_ON(stream->sample_flags != props->sample_flags)) { 1710 ret = -ENODEV; 1711 goto err_flags; 1712 } 1713 1714 list_add(&stream->link, &dev_priv->perf.streams); 1715 1716 if (param->flags & I915_PERF_FLAG_FD_CLOEXEC) 1717 f_flags |= O_CLOEXEC; 1718 if (param->flags & I915_PERF_FLAG_FD_NONBLOCK) 1719 f_flags |= O_NONBLOCK; 1720 1721 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); 1722 if (stream_fd < 0) { 1723 ret = stream_fd; 1724 goto err_open; 1725 } 1726 1727 if (!(param->flags & I915_PERF_FLAG_DISABLED)) 1728 i915_perf_enable_locked(stream); 1729 1730 return stream_fd; 1731 1732 err_open: 1733 list_del(&stream->link); 1734 err_flags: 1735 if (stream->ops->destroy) 1736 stream->ops->destroy(stream); 1737 err_alloc: 1738 kfree(stream); 1739 err_ctx: 1740 if (specific_ctx) 1741 i915_gem_context_put_unlocked(specific_ctx); 1742 err: 1743 return ret; 1744 } 1745 1746 /** 1747 * read_properties_unlocked - validate + copy userspace stream open properties 1748 * @dev_priv: i915 device instance 1749 * @uprops: The array of u64 key value pairs given by userspace 1750 * @n_props: The number of key value pairs expected in @uprops 1751 * @props: The stream configuration built up while validating properties 1752 * 1753 * Note this function only validates properties in isolation it doesn't 1754 * validate that the combination of properties makes sense or that all 1755 * properties necessary for a particular kind of stream have been set. 1756 * 1757 * Note that there currently aren't any ordering requirements for properties so 1758 * we shouldn't validate or assume anything about ordering here. This doesn't 1759 * rule out defining new properties with ordering requirements in the future. 1760 */ 1761 static int read_properties_unlocked(struct drm_i915_private *dev_priv, 1762 u64 __user *uprops, 1763 u32 n_props, 1764 struct perf_open_properties *props) 1765 { 1766 u64 __user *uprop = uprops; 1767 int i; 1768 1769 memset(props, 0, sizeof(struct perf_open_properties)); 1770 1771 if (!n_props) { 1772 DRM_DEBUG("No i915 perf properties given\n"); 1773 return -EINVAL; 1774 } 1775 1776 /* Considering that ID = 0 is reserved and assuming that we don't 1777 * (currently) expect any configurations to ever specify duplicate 1778 * values for a particular property ID then the last _PROP_MAX value is 1779 * one greater than the maximum number of properties we expect to get 1780 * from userspace. 1781 */ 1782 if (n_props >= DRM_I915_PERF_PROP_MAX) { 1783 DRM_DEBUG("More i915 perf properties specified than exist\n"); 1784 return -EINVAL; 1785 } 1786 1787 for (i = 0; i < n_props; i++) { 1788 u64 oa_period, oa_freq_hz; 1789 u64 id, value; 1790 int ret; 1791 1792 ret = get_user(id, uprop); 1793 if (ret) 1794 return ret; 1795 1796 ret = get_user(value, uprop + 1); 1797 if (ret) 1798 return ret; 1799 1800 if (id == 0 || id >= DRM_I915_PERF_PROP_MAX) { 1801 DRM_DEBUG("Unknown i915 perf property ID\n"); 1802 return -EINVAL; 1803 } 1804 1805 switch ((enum drm_i915_perf_property_id)id) { 1806 case DRM_I915_PERF_PROP_CTX_HANDLE: 1807 props->single_context = 1; 1808 props->ctx_handle = value; 1809 break; 1810 case DRM_I915_PERF_PROP_SAMPLE_OA: 1811 props->sample_flags |= SAMPLE_OA_REPORT; 1812 break; 1813 case DRM_I915_PERF_PROP_OA_METRICS_SET: 1814 if (value == 0 || 1815 value > dev_priv->perf.oa.n_builtin_sets) { 1816 DRM_DEBUG("Unknown OA metric set ID\n"); 1817 return -EINVAL; 1818 } 1819 props->metrics_set = value; 1820 break; 1821 case DRM_I915_PERF_PROP_OA_FORMAT: 1822 if (value == 0 || value >= I915_OA_FORMAT_MAX) { 1823 DRM_DEBUG("Invalid OA report format\n"); 1824 return -EINVAL; 1825 } 1826 if (!dev_priv->perf.oa.oa_formats[value].size) { 1827 DRM_DEBUG("Invalid OA report format\n"); 1828 return -EINVAL; 1829 } 1830 props->oa_format = value; 1831 break; 1832 case DRM_I915_PERF_PROP_OA_EXPONENT: 1833 if (value > OA_EXPONENT_MAX) { 1834 DRM_DEBUG("OA timer exponent too high (> %u)\n", 1835 OA_EXPONENT_MAX); 1836 return -EINVAL; 1837 } 1838 1839 /* Theoretically we can program the OA unit to sample 1840 * every 160ns but don't allow that by default unless 1841 * root. 1842 * 1843 * On Haswell the period is derived from the exponent 1844 * as: 1845 * 1846 * period = 80ns * 2^(exponent + 1) 1847 */ 1848 BUILD_BUG_ON(sizeof(oa_period) != 8); 1849 oa_period = 80ull * (2ull << value); 1850 1851 /* This check is primarily to ensure that oa_period <= 1852 * UINT32_MAX (before passing to do_div which only 1853 * accepts a u32 denominator), but we can also skip 1854 * checking anything < 1Hz which implicitly can't be 1855 * limited via an integer oa_max_sample_rate. 1856 */ 1857 if (oa_period <= NSEC_PER_SEC) { 1858 u64 tmp = NSEC_PER_SEC; 1859 do_div(tmp, oa_period); 1860 oa_freq_hz = tmp; 1861 } else 1862 oa_freq_hz = 0; 1863 1864 if (oa_freq_hz > i915_oa_max_sample_rate && 1865 !capable(CAP_SYS_ADMIN)) { 1866 DRM_DEBUG("OA exponent would exceed the max sampling frequency (sysctl dev.i915.oa_max_sample_rate) %uHz without root privileges\n", 1867 i915_oa_max_sample_rate); 1868 return -EACCES; 1869 } 1870 1871 props->oa_periodic = true; 1872 props->oa_period_exponent = value; 1873 break; 1874 case DRM_I915_PERF_PROP_MAX: 1875 MISSING_CASE(id); 1876 return -EINVAL; 1877 } 1878 1879 uprop += 2; 1880 } 1881 1882 return 0; 1883 } 1884 #endif 1885 1886 /** 1887 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD 1888 * @dev: drm device 1889 * @data: ioctl data copied from userspace (unvalidated) 1890 * @file: drm file 1891 * 1892 * Validates the stream open parameters given by userspace including flags 1893 * and an array of u64 key, value pair properties. 1894 * 1895 * Very little is assumed up front about the nature of the stream being 1896 * opened (for instance we don't assume it's for periodic OA unit metrics). An 1897 * i915-perf stream is expected to be a suitable interface for other forms of 1898 * buffered data written by the GPU besides periodic OA metrics. 1899 * 1900 * Note we copy the properties from userspace outside of the i915 perf 1901 * mutex to avoid an awkward lockdep with mmap_sem. 1902 * 1903 * Most of the implementation details are handled by 1904 * i915_perf_open_ioctl_locked() after taking the &drm_i915_private->perf.lock 1905 * mutex for serializing with any non-file-operation driver hooks. 1906 * 1907 * Return: A newly opened i915 Perf stream file descriptor or negative 1908 * error code on failure. 1909 */ 1910 int i915_perf_open_ioctl(struct drm_device *dev, void *data, 1911 struct drm_file *file) 1912 { 1913 #if 0 1914 struct drm_i915_private *dev_priv = dev->dev_private; 1915 struct drm_i915_perf_open_param *param = data; 1916 struct perf_open_properties props; 1917 u32 known_open_flags; 1918 int ret; 1919 1920 if (!dev_priv->perf.initialized) { 1921 #endif 1922 DRM_DEBUG("i915 perf interface not available for this system\n"); 1923 return -ENOTSUPP; 1924 #if 0 1925 } 1926 1927 known_open_flags = I915_PERF_FLAG_FD_CLOEXEC | 1928 I915_PERF_FLAG_FD_NONBLOCK | 1929 I915_PERF_FLAG_DISABLED; 1930 if (param->flags & ~known_open_flags) { 1931 DRM_DEBUG("Unknown drm_i915_perf_open_param flag\n"); 1932 return -EINVAL; 1933 } 1934 1935 ret = read_properties_unlocked(dev_priv, 1936 u64_to_user_ptr(param->properties_ptr), 1937 param->num_properties, 1938 &props); 1939 if (ret) 1940 return ret; 1941 1942 mutex_lock(&dev_priv->perf.lock); 1943 ret = i915_perf_open_ioctl_locked(dev_priv, param, &props, file); 1944 mutex_unlock(&dev_priv->perf.lock); 1945 1946 return ret; 1947 #endif 1948 } 1949 1950 /** 1951 * i915_perf_register - exposes i915-perf to userspace 1952 * @dev_priv: i915 device instance 1953 * 1954 * In particular OA metric sets are advertised under a sysfs metrics/ 1955 * directory allowing userspace to enumerate valid IDs that can be 1956 * used to open an i915-perf stream. 1957 */ 1958 void i915_perf_register(struct drm_i915_private *dev_priv) 1959 { 1960 if (!IS_HASWELL(dev_priv)) 1961 return; 1962 1963 if (!dev_priv->perf.initialized) 1964 return; 1965 1966 #if 0 1967 /* To be sure we're synchronized with an attempted 1968 * i915_perf_open_ioctl(); considering that we register after 1969 * being exposed to userspace. 1970 */ 1971 mutex_lock(&dev_priv->perf.lock); 1972 1973 dev_priv->perf.metrics_kobj = 1974 kobject_create_and_add("metrics", 1975 &dev_priv->drm.primary->kdev->kobj); 1976 if (!dev_priv->perf.metrics_kobj) 1977 goto exit; 1978 1979 if (i915_perf_register_sysfs_hsw(dev_priv)) { 1980 kobject_put(dev_priv->perf.metrics_kobj); 1981 dev_priv->perf.metrics_kobj = NULL; 1982 } 1983 1984 exit: 1985 mutex_unlock(&dev_priv->perf.lock); 1986 #endif 1987 } 1988 1989 /** 1990 * i915_perf_unregister - hide i915-perf from userspace 1991 * @dev_priv: i915 device instance 1992 * 1993 * i915-perf state cleanup is split up into an 'unregister' and 1994 * 'deinit' phase where the interface is first hidden from 1995 * userspace by i915_perf_unregister() before cleaning up 1996 * remaining state in i915_perf_fini(). 1997 */ 1998 void i915_perf_unregister(struct drm_i915_private *dev_priv) 1999 { 2000 if (!IS_HASWELL(dev_priv)) 2001 return; 2002 2003 if (!dev_priv->perf.metrics_kobj) 2004 return; 2005 2006 #if 0 2007 i915_perf_unregister_sysfs_hsw(dev_priv); 2008 2009 kobject_put(dev_priv->perf.metrics_kobj); 2010 dev_priv->perf.metrics_kobj = NULL; 2011 #endif 2012 } 2013 2014 #if 0 2015 static struct ctl_table oa_table[] = { 2016 { 2017 .procname = "perf_stream_paranoid", 2018 .data = &i915_perf_stream_paranoid, 2019 .maxlen = sizeof(i915_perf_stream_paranoid), 2020 .mode = 0644, 2021 .proc_handler = proc_dointvec_minmax, 2022 .extra1 = &zero, 2023 .extra2 = &one, 2024 }, 2025 { 2026 .procname = "oa_max_sample_rate", 2027 .data = &i915_oa_max_sample_rate, 2028 .maxlen = sizeof(i915_oa_max_sample_rate), 2029 .mode = 0644, 2030 .proc_handler = proc_dointvec_minmax, 2031 .extra1 = &zero, 2032 .extra2 = &oa_sample_rate_hard_limit, 2033 }, 2034 {} 2035 }; 2036 2037 static struct ctl_table i915_root[] = { 2038 { 2039 .procname = "i915", 2040 .maxlen = 0, 2041 .mode = 0555, 2042 .child = oa_table, 2043 }, 2044 {} 2045 }; 2046 2047 static struct ctl_table dev_root[] = { 2048 { 2049 .procname = "dev", 2050 .maxlen = 0, 2051 .mode = 0555, 2052 .child = i915_root, 2053 }, 2054 {} 2055 }; 2056 #endif 2057 2058 /** 2059 * i915_perf_init - initialize i915-perf state on module load 2060 * @dev_priv: i915 device instance 2061 * 2062 * Initializes i915-perf state without exposing anything to userspace. 2063 * 2064 * Note: i915-perf initialization is split into an 'init' and 'register' 2065 * phase with the i915_perf_register() exposing state to userspace. 2066 */ 2067 void i915_perf_init(struct drm_i915_private *dev_priv) 2068 { 2069 if (!IS_HASWELL(dev_priv)) 2070 return; 2071 2072 #if 0 2073 hrtimer_init(&dev_priv->perf.oa.poll_check_timer, 2074 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 2075 dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb; 2076 init_waitqueue_head(&dev_priv->perf.oa.poll_wq); 2077 2078 INIT_LIST_HEAD(&dev_priv->perf.streams); 2079 mutex_init(&dev_priv->perf.lock); 2080 spin_lock_init(&dev_priv->perf.hook_lock); 2081 2082 dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer; 2083 dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set; 2084 dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set; 2085 dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable; 2086 dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable; 2087 dev_priv->perf.oa.ops.read = gen7_oa_read; 2088 dev_priv->perf.oa.ops.oa_buffer_is_empty = 2089 gen7_oa_buffer_is_empty_fop_unlocked; 2090 2091 dev_priv->perf.oa.timestamp_frequency = 12500000; 2092 2093 dev_priv->perf.oa.oa_formats = hsw_oa_formats; 2094 2095 dev_priv->perf.oa.n_builtin_sets = 2096 i915_oa_n_builtin_metric_sets_hsw; 2097 2098 dev_priv->perf.sysctl_header = register_sysctl_table(dev_root); 2099 #endif 2100 2101 dev_priv->perf.initialized = true; 2102 } 2103 2104 /** 2105 * i915_perf_fini - Counter part to i915_perf_init() 2106 * @dev_priv: i915 device instance 2107 */ 2108 void i915_perf_fini(struct drm_i915_private *dev_priv) 2109 { 2110 if (!dev_priv->perf.initialized) 2111 return; 2112 2113 #if 0 2114 unregister_sysctl_table(dev_priv->perf.sysctl_header); 2115 2116 memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops)); 2117 #endif 2118 dev_priv->perf.initialized = false; 2119 } 2120