1// Copyright 2018 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package procfs 15 16// While implementing parsing of /proc/[pid]/mountstats, this blog was used 17// heavily as a reference: 18// https://utcc.utoronto.ca/~cks/space/blog/linux/NFSMountstatsIndex 19// 20// Special thanks to Chris Siebenmann for all of his posts explaining the 21// various statistics available for NFS. 22 23import ( 24 "bufio" 25 "fmt" 26 "io" 27 "strconv" 28 "strings" 29 "time" 30) 31 32// Constants shared between multiple functions. 33const ( 34 deviceEntryLen = 8 35 36 fieldBytesLen = 8 37 fieldEventsLen = 27 38 39 statVersion10 = "1.0" 40 statVersion11 = "1.1" 41 42 fieldTransport10TCPLen = 10 43 fieldTransport10UDPLen = 7 44 45 fieldTransport11TCPLen = 13 46 fieldTransport11UDPLen = 10 47) 48 49// A Mount is a device mount parsed from /proc/[pid]/mountstats. 50type Mount struct { 51 // Name of the device. 52 Device string 53 // The mount point of the device. 54 Mount string 55 // The filesystem type used by the device. 56 Type string 57 // If available additional statistics related to this Mount. 58 // Use a type assertion to determine if additional statistics are available. 59 Stats MountStats 60} 61 62// A MountStats is a type which contains detailed statistics for a specific 63// type of Mount. 64type MountStats interface { 65 mountStats() 66} 67 68// A MountStatsNFS is a MountStats implementation for NFSv3 and v4 mounts. 69type MountStatsNFS struct { 70 // The version of statistics provided. 71 StatVersion string 72 // The mount options of the NFS mount. 73 Opts map[string]string 74 // The age of the NFS mount. 75 Age time.Duration 76 // Statistics related to byte counters for various operations. 77 Bytes NFSBytesStats 78 // Statistics related to various NFS event occurrences. 79 Events NFSEventsStats 80 // Statistics broken down by filesystem operation. 81 Operations []NFSOperationStats 82 // Statistics about the NFS RPC transport. 83 Transport NFSTransportStats 84} 85 86// mountStats implements MountStats. 87func (m MountStatsNFS) mountStats() {} 88 89// A NFSBytesStats contains statistics about the number of bytes read and written 90// by an NFS client to and from an NFS server. 91type NFSBytesStats struct { 92 // Number of bytes read using the read() syscall. 93 Read uint64 94 // Number of bytes written using the write() syscall. 95 Write uint64 96 // Number of bytes read using the read() syscall in O_DIRECT mode. 97 DirectRead uint64 98 // Number of bytes written using the write() syscall in O_DIRECT mode. 99 DirectWrite uint64 100 // Number of bytes read from the NFS server, in total. 101 ReadTotal uint64 102 // Number of bytes written to the NFS server, in total. 103 WriteTotal uint64 104 // Number of pages read directly via mmap()'d files. 105 ReadPages uint64 106 // Number of pages written directly via mmap()'d files. 107 WritePages uint64 108} 109 110// A NFSEventsStats contains statistics about NFS event occurrences. 111type NFSEventsStats struct { 112 // Number of times cached inode attributes are re-validated from the server. 113 InodeRevalidate uint64 114 // Number of times cached dentry nodes are re-validated from the server. 115 DnodeRevalidate uint64 116 // Number of times an inode cache is cleared. 117 DataInvalidate uint64 118 // Number of times cached inode attributes are invalidated. 119 AttributeInvalidate uint64 120 // Number of times files or directories have been open()'d. 121 VFSOpen uint64 122 // Number of times a directory lookup has occurred. 123 VFSLookup uint64 124 // Number of times permissions have been checked. 125 VFSAccess uint64 126 // Number of updates (and potential writes) to pages. 127 VFSUpdatePage uint64 128 // Number of pages read directly via mmap()'d files. 129 VFSReadPage uint64 130 // Number of times a group of pages have been read. 131 VFSReadPages uint64 132 // Number of pages written directly via mmap()'d files. 133 VFSWritePage uint64 134 // Number of times a group of pages have been written. 135 VFSWritePages uint64 136 // Number of times directory entries have been read with getdents(). 137 VFSGetdents uint64 138 // Number of times attributes have been set on inodes. 139 VFSSetattr uint64 140 // Number of pending writes that have been forcefully flushed to the server. 141 VFSFlush uint64 142 // Number of times fsync() has been called on directories and files. 143 VFSFsync uint64 144 // Number of times locking has been attempted on a file. 145 VFSLock uint64 146 // Number of times files have been closed and released. 147 VFSFileRelease uint64 148 // Unknown. Possibly unused. 149 CongestionWait uint64 150 // Number of times files have been truncated. 151 Truncation uint64 152 // Number of times a file has been grown due to writes beyond its existing end. 153 WriteExtension uint64 154 // Number of times a file was removed while still open by another process. 155 SillyRename uint64 156 // Number of times the NFS server gave less data than expected while reading. 157 ShortRead uint64 158 // Number of times the NFS server wrote less data than expected while writing. 159 ShortWrite uint64 160 // Number of times the NFS server indicated EJUKEBOX; retrieving data from 161 // offline storage. 162 JukeboxDelay uint64 163 // Number of NFS v4.1+ pNFS reads. 164 PNFSRead uint64 165 // Number of NFS v4.1+ pNFS writes. 166 PNFSWrite uint64 167} 168 169// A NFSOperationStats contains statistics for a single operation. 170type NFSOperationStats struct { 171 // The name of the operation. 172 Operation string 173 // Number of requests performed for this operation. 174 Requests uint64 175 // Number of times an actual RPC request has been transmitted for this operation. 176 Transmissions uint64 177 // Number of times a request has had a major timeout. 178 MajorTimeouts uint64 179 // Number of bytes sent for this operation, including RPC headers and payload. 180 BytesSent uint64 181 // Number of bytes received for this operation, including RPC headers and payload. 182 BytesReceived uint64 183 // Duration all requests spent queued for transmission before they were sent. 184 CumulativeQueueMilliseconds uint64 185 // Duration it took to get a reply back after the request was transmitted. 186 CumulativeTotalResponseMilliseconds uint64 187 // Duration from when a request was enqueued to when it was completely handled. 188 CumulativeTotalRequestMilliseconds uint64 189 // The count of operations that complete with tk_status < 0. These statuses usually indicate error conditions. 190 Errors uint64 191} 192 193// A NFSTransportStats contains statistics for the NFS mount RPC requests and 194// responses. 195type NFSTransportStats struct { 196 // The transport protocol used for the NFS mount. 197 Protocol string 198 // The local port used for the NFS mount. 199 Port uint64 200 // Number of times the client has had to establish a connection from scratch 201 // to the NFS server. 202 Bind uint64 203 // Number of times the client has made a TCP connection to the NFS server. 204 Connect uint64 205 // Duration (in jiffies, a kernel internal unit of time) the NFS mount has 206 // spent waiting for connections to the server to be established. 207 ConnectIdleTime uint64 208 // Duration since the NFS mount last saw any RPC traffic. 209 IdleTimeSeconds uint64 210 // Number of RPC requests for this mount sent to the NFS server. 211 Sends uint64 212 // Number of RPC responses for this mount received from the NFS server. 213 Receives uint64 214 // Number of times the NFS server sent a response with a transaction ID 215 // unknown to this client. 216 BadTransactionIDs uint64 217 // A running counter, incremented on each request as the current difference 218 // ebetween sends and receives. 219 CumulativeActiveRequests uint64 220 // A running counter, incremented on each request by the current backlog 221 // queue size. 222 CumulativeBacklog uint64 223 224 // Stats below only available with stat version 1.1. 225 226 // Maximum number of simultaneously active RPC requests ever used. 227 MaximumRPCSlotsUsed uint64 228 // A running counter, incremented on each request as the current size of the 229 // sending queue. 230 CumulativeSendingQueue uint64 231 // A running counter, incremented on each request as the current size of the 232 // pending queue. 233 CumulativePendingQueue uint64 234} 235 236// parseMountStats parses a /proc/[pid]/mountstats file and returns a slice 237// of Mount structures containing detailed information about each mount. 238// If available, statistics for each mount are parsed as well. 239func parseMountStats(r io.Reader) ([]*Mount, error) { 240 const ( 241 device = "device" 242 statVersionPrefix = "statvers=" 243 244 nfs3Type = "nfs" 245 nfs4Type = "nfs4" 246 ) 247 248 var mounts []*Mount 249 250 s := bufio.NewScanner(r) 251 for s.Scan() { 252 // Only look for device entries in this function 253 ss := strings.Fields(string(s.Bytes())) 254 if len(ss) == 0 || ss[0] != device { 255 continue 256 } 257 258 m, err := parseMount(ss) 259 if err != nil { 260 return nil, err 261 } 262 263 // Does this mount also possess statistics information? 264 if len(ss) > deviceEntryLen { 265 // Only NFSv3 and v4 are supported for parsing statistics 266 if m.Type != nfs3Type && m.Type != nfs4Type { 267 return nil, fmt.Errorf("cannot parse MountStats for fstype %q", m.Type) 268 } 269 270 statVersion := strings.TrimPrefix(ss[8], statVersionPrefix) 271 272 stats, err := parseMountStatsNFS(s, statVersion) 273 if err != nil { 274 return nil, err 275 } 276 277 m.Stats = stats 278 } 279 280 mounts = append(mounts, m) 281 } 282 283 return mounts, s.Err() 284} 285 286// parseMount parses an entry in /proc/[pid]/mountstats in the format: 287// device [device] mounted on [mount] with fstype [type] 288func parseMount(ss []string) (*Mount, error) { 289 if len(ss) < deviceEntryLen { 290 return nil, fmt.Errorf("invalid device entry: %v", ss) 291 } 292 293 // Check for specific words appearing at specific indices to ensure 294 // the format is consistent with what we expect 295 format := []struct { 296 i int 297 s string 298 }{ 299 {i: 0, s: "device"}, 300 {i: 2, s: "mounted"}, 301 {i: 3, s: "on"}, 302 {i: 5, s: "with"}, 303 {i: 6, s: "fstype"}, 304 } 305 306 for _, f := range format { 307 if ss[f.i] != f.s { 308 return nil, fmt.Errorf("invalid device entry: %v", ss) 309 } 310 } 311 312 return &Mount{ 313 Device: ss[1], 314 Mount: ss[4], 315 Type: ss[7], 316 }, nil 317} 318 319// parseMountStatsNFS parses a MountStatsNFS by scanning additional information 320// related to NFS statistics. 321func parseMountStatsNFS(s *bufio.Scanner, statVersion string) (*MountStatsNFS, error) { 322 // Field indicators for parsing specific types of data 323 const ( 324 fieldOpts = "opts:" 325 fieldAge = "age:" 326 fieldBytes = "bytes:" 327 fieldEvents = "events:" 328 fieldPerOpStats = "per-op" 329 fieldTransport = "xprt:" 330 ) 331 332 stats := &MountStatsNFS{ 333 StatVersion: statVersion, 334 } 335 336 for s.Scan() { 337 ss := strings.Fields(string(s.Bytes())) 338 if len(ss) == 0 { 339 break 340 } 341 if len(ss) < 2 { 342 return nil, fmt.Errorf("not enough information for NFS stats: %v", ss) 343 } 344 345 switch ss[0] { 346 case fieldOpts: 347 if stats.Opts == nil { 348 stats.Opts = map[string]string{} 349 } 350 for _, opt := range strings.Split(ss[1], ",") { 351 split := strings.Split(opt, "=") 352 if len(split) == 2 { 353 stats.Opts[split[0]] = split[1] 354 } else { 355 stats.Opts[opt] = "" 356 } 357 } 358 case fieldAge: 359 // Age integer is in seconds 360 d, err := time.ParseDuration(ss[1] + "s") 361 if err != nil { 362 return nil, err 363 } 364 365 stats.Age = d 366 case fieldBytes: 367 bstats, err := parseNFSBytesStats(ss[1:]) 368 if err != nil { 369 return nil, err 370 } 371 372 stats.Bytes = *bstats 373 case fieldEvents: 374 estats, err := parseNFSEventsStats(ss[1:]) 375 if err != nil { 376 return nil, err 377 } 378 379 stats.Events = *estats 380 case fieldTransport: 381 if len(ss) < 3 { 382 return nil, fmt.Errorf("not enough information for NFS transport stats: %v", ss) 383 } 384 385 tstats, err := parseNFSTransportStats(ss[1:], statVersion) 386 if err != nil { 387 return nil, err 388 } 389 390 stats.Transport = *tstats 391 } 392 393 // When encountering "per-operation statistics", we must break this 394 // loop and parse them separately to ensure we can terminate parsing 395 // before reaching another device entry; hence why this 'if' statement 396 // is not just another switch case 397 if ss[0] == fieldPerOpStats { 398 break 399 } 400 } 401 402 if err := s.Err(); err != nil { 403 return nil, err 404 } 405 406 // NFS per-operation stats appear last before the next device entry 407 perOpStats, err := parseNFSOperationStats(s) 408 if err != nil { 409 return nil, err 410 } 411 412 stats.Operations = perOpStats 413 414 return stats, nil 415} 416 417// parseNFSBytesStats parses a NFSBytesStats line using an input set of 418// integer fields. 419func parseNFSBytesStats(ss []string) (*NFSBytesStats, error) { 420 if len(ss) != fieldBytesLen { 421 return nil, fmt.Errorf("invalid NFS bytes stats: %v", ss) 422 } 423 424 ns := make([]uint64, 0, fieldBytesLen) 425 for _, s := range ss { 426 n, err := strconv.ParseUint(s, 10, 64) 427 if err != nil { 428 return nil, err 429 } 430 431 ns = append(ns, n) 432 } 433 434 return &NFSBytesStats{ 435 Read: ns[0], 436 Write: ns[1], 437 DirectRead: ns[2], 438 DirectWrite: ns[3], 439 ReadTotal: ns[4], 440 WriteTotal: ns[5], 441 ReadPages: ns[6], 442 WritePages: ns[7], 443 }, nil 444} 445 446// parseNFSEventsStats parses a NFSEventsStats line using an input set of 447// integer fields. 448func parseNFSEventsStats(ss []string) (*NFSEventsStats, error) { 449 if len(ss) != fieldEventsLen { 450 return nil, fmt.Errorf("invalid NFS events stats: %v", ss) 451 } 452 453 ns := make([]uint64, 0, fieldEventsLen) 454 for _, s := range ss { 455 n, err := strconv.ParseUint(s, 10, 64) 456 if err != nil { 457 return nil, err 458 } 459 460 ns = append(ns, n) 461 } 462 463 return &NFSEventsStats{ 464 InodeRevalidate: ns[0], 465 DnodeRevalidate: ns[1], 466 DataInvalidate: ns[2], 467 AttributeInvalidate: ns[3], 468 VFSOpen: ns[4], 469 VFSLookup: ns[5], 470 VFSAccess: ns[6], 471 VFSUpdatePage: ns[7], 472 VFSReadPage: ns[8], 473 VFSReadPages: ns[9], 474 VFSWritePage: ns[10], 475 VFSWritePages: ns[11], 476 VFSGetdents: ns[12], 477 VFSSetattr: ns[13], 478 VFSFlush: ns[14], 479 VFSFsync: ns[15], 480 VFSLock: ns[16], 481 VFSFileRelease: ns[17], 482 CongestionWait: ns[18], 483 Truncation: ns[19], 484 WriteExtension: ns[20], 485 SillyRename: ns[21], 486 ShortRead: ns[22], 487 ShortWrite: ns[23], 488 JukeboxDelay: ns[24], 489 PNFSRead: ns[25], 490 PNFSWrite: ns[26], 491 }, nil 492} 493 494// parseNFSOperationStats parses a slice of NFSOperationStats by scanning 495// additional information about per-operation statistics until an empty 496// line is reached. 497func parseNFSOperationStats(s *bufio.Scanner) ([]NFSOperationStats, error) { 498 const ( 499 // Minimum number of expected fields in each per-operation statistics set 500 minFields = 9 501 ) 502 503 var ops []NFSOperationStats 504 505 for s.Scan() { 506 ss := strings.Fields(string(s.Bytes())) 507 if len(ss) == 0 { 508 // Must break when reading a blank line after per-operation stats to 509 // enable top-level function to parse the next device entry 510 break 511 } 512 513 if len(ss) < minFields { 514 return nil, fmt.Errorf("invalid NFS per-operations stats: %v", ss) 515 } 516 517 // Skip string operation name for integers 518 ns := make([]uint64, 0, minFields-1) 519 for _, st := range ss[1:] { 520 n, err := strconv.ParseUint(st, 10, 64) 521 if err != nil { 522 return nil, err 523 } 524 525 ns = append(ns, n) 526 } 527 528 opStats := NFSOperationStats{ 529 Operation: strings.TrimSuffix(ss[0], ":"), 530 Requests: ns[0], 531 Transmissions: ns[1], 532 MajorTimeouts: ns[2], 533 BytesSent: ns[3], 534 BytesReceived: ns[4], 535 CumulativeQueueMilliseconds: ns[5], 536 CumulativeTotalResponseMilliseconds: ns[6], 537 CumulativeTotalRequestMilliseconds: ns[7], 538 } 539 540 if len(ns) > 8 { 541 opStats.Errors = ns[8] 542 } 543 544 ops = append(ops, opStats) 545 } 546 547 return ops, s.Err() 548} 549 550// parseNFSTransportStats parses a NFSTransportStats line using an input set of 551// integer fields matched to a specific stats version. 552func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats, error) { 553 // Extract the protocol field. It is the only string value in the line 554 protocol := ss[0] 555 ss = ss[1:] 556 557 switch statVersion { 558 case statVersion10: 559 var expectedLength int 560 if protocol == "tcp" { 561 expectedLength = fieldTransport10TCPLen 562 } else if protocol == "udp" { 563 expectedLength = fieldTransport10UDPLen 564 } else { 565 return nil, fmt.Errorf("invalid NFS protocol \"%s\" in stats 1.0 statement: %v", protocol, ss) 566 } 567 if len(ss) != expectedLength { 568 return nil, fmt.Errorf("invalid NFS transport stats 1.0 statement: %v", ss) 569 } 570 case statVersion11: 571 var expectedLength int 572 if protocol == "tcp" { 573 expectedLength = fieldTransport11TCPLen 574 } else if protocol == "udp" { 575 expectedLength = fieldTransport11UDPLen 576 } else { 577 return nil, fmt.Errorf("invalid NFS protocol \"%s\" in stats 1.1 statement: %v", protocol, ss) 578 } 579 if len(ss) != expectedLength { 580 return nil, fmt.Errorf("invalid NFS transport stats 1.1 statement: %v", ss) 581 } 582 default: 583 return nil, fmt.Errorf("unrecognized NFS transport stats version: %q", statVersion) 584 } 585 586 // Allocate enough for v1.1 stats since zero value for v1.1 stats will be okay 587 // in a v1.0 response. Since the stat length is bigger for TCP stats, we use 588 // the TCP length here. 589 // 590 // Note: slice length must be set to length of v1.1 stats to avoid a panic when 591 // only v1.0 stats are present. 592 // See: https://github.com/prometheus/node_exporter/issues/571. 593 ns := make([]uint64, fieldTransport11TCPLen) 594 for i, s := range ss { 595 n, err := strconv.ParseUint(s, 10, 64) 596 if err != nil { 597 return nil, err 598 } 599 600 ns[i] = n 601 } 602 603 // The fields differ depending on the transport protocol (TCP or UDP) 604 // From https://utcc.utoronto.ca/%7Ecks/space/blog/linux/NFSMountstatsXprt 605 // 606 // For the udp RPC transport there is no connection count, connect idle time, 607 // or idle time (fields #3, #4, and #5); all other fields are the same. So 608 // we set them to 0 here. 609 if protocol == "udp" { 610 ns = append(ns[:2], append(make([]uint64, 3), ns[2:]...)...) 611 } 612 613 return &NFSTransportStats{ 614 Protocol: protocol, 615 Port: ns[0], 616 Bind: ns[1], 617 Connect: ns[2], 618 ConnectIdleTime: ns[3], 619 IdleTimeSeconds: ns[4], 620 Sends: ns[5], 621 Receives: ns[6], 622 BadTransactionIDs: ns[7], 623 CumulativeActiveRequests: ns[8], 624 CumulativeBacklog: ns[9], 625 MaximumRPCSlotsUsed: ns[10], 626 CumulativeSendingQueue: ns[11], 627 CumulativePendingQueue: ns[12], 628 }, nil 629} 630