1.. -*- mode: rst; -*- 2 3.. |json-status-format| replace:: 4 "cluster":{ 5 "layers":{ 6 "_valid":true, 7 "_error":"some error description" 8 }, 9 "processes":{ 10 "$map_key=zoneid":{ 11 "version":"3.0.0", // a process version will not be reported if it is not protocol-compatible; it will be absent from status 12 "machine_id":"0ccb4e0feddb5583010f6b77d9d10ece", 13 "locality":{ // This will contain any locality fields that are provided on the command line 14 "$map_key=localityName":"value" 15 }, 16 "class_source":{ 17 "$enum":[ 18 "command_line", 19 "configure_auto", 20 "set_class" 21 ] 22 }, 23 "class_type":{ 24 "$enum":[ 25 "unset", 26 "storage", 27 "transaction", 28 "resolution", 29 "proxy", 30 "master", 31 "test" 32 ] 33 }, 34 "degraded":true, 35 "roles":[ 36 { 37 "query_queue_max":0, 38 "input_bytes":{ 39 "hz":0.0, 40 "counter":0, 41 "roughness":0.0 42 }, 43 "stored_bytes":12341234, 44 "kvstore_used_bytes":12341234, 45 "kvstore_available_bytes":12341234, 46 "kvstore_free_bytes":12341234, 47 "kvstore_total_bytes":12341234, 48 "durable_bytes":{ 49 "hz":0.0, 50 "counter":0, 51 "roughness":0.0 52 }, 53 "queue_disk_used_bytes":12341234, 54 "queue_disk_available_bytes":12341234, 55 "queue_disk_free_bytes":12341234, 56 "queue_disk_total_bytes":12341234, 57 "role":{ 58 "$enum":[ 59 "master", 60 "proxy", 61 "log", 62 "storage", 63 "resolver", 64 "cluster_controller", 65 "data_distributor", 66 "ratekeeper" 67 ] 68 }, 69 "data_version":12341234, 70 "durable_version":12341234, 71 "data_lag": { 72 "seconds":5.0, 73 "versions":12341234 74 }, 75 "durability_lag": { 76 "seconds":5.0, 77 "versions":12341234 78 }, 79 "id":"eb84471d68c12d1d26f692a50000003f", 80 "total_queries":{ 81 "hz":0.0, 82 "counter":0, 83 "roughness":0.0 84 }, 85 "finished_queries":{ 86 "hz":0.0, 87 "counter":0, 88 "roughness":0.0 89 }, 90 "bytes_queried":{ 91 "hz":0.0, 92 "counter":0, 93 "roughness":0.0 94 }, 95 "keys_queried":{ 96 "hz":0.0, 97 "counter":0, 98 "roughness":0.0 99 }, 100 "mutation_bytes":{ 101 "hz":0.0, 102 "counter":0, 103 "roughness":0.0 104 }, 105 "mutations":{ 106 "hz":0.0, 107 "counter":0, 108 "roughness":0.0 109 }, 110 "grv_latency_bands":{ // How many GRV requests belong to the latency (in seconds) band (e.g., How many requests belong to [0.01,0.1] latency band). The key is the upper bound of the band and the lower bound is the next smallest band (or 0, if none). Example: {0.01: 27, 0.1: 18, 1: 1, inf: 98,filtered: 10}, we have 18 requests in [0.01, 0.1) band. 111 "$map_key=upperBoundOfBand": 1 112 }, 113 "read_latency_bands":{ 114 "$map_key=upperBoundOfBand": 1 115 }, 116 "commit_latency_bands":{ 117 "$map_key=upperBoundOfBand": 1 118 } 119 } 120 ], 121 "command_line":"-r simulation", 122 "memory":{ 123 "available_bytes":0, //an estimate of the process' fair share of the memory available to fdbservers 124 "limit_bytes":0, // memory limit per process 125 "unused_allocated_memory":0, 126 "used_bytes":0 127 }, 128 "messages":[ 129 { 130 "time":12345.12312, 131 "type":"x", 132 "name":{ // when not limiting 133 "$enum":[ 134 "file_open_error", 135 "incorrect_cluster_file_contents", 136 "process_error", 137 "io_error", 138 "io_timeout", 139 "platform_error", 140 "storage_server_lagging", 141 "(other FDB error messages)" 142 ] 143 }, 144 "raw_log_message":"<stuff/>", 145 "description":"abc" 146 } 147 ], 148 "fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece", 149 "under_maintenance":true, 150 "excluded":false, 151 "address":"1.2.3.4:1234", 152 "disk":{ 153 "free_bytes":3451233456234, // an estimate of how many bytes are free to allocate to fdbservers without swapping 154 "reads":{ 155 "hz":0.0, 156 "counter":0, 157 "sectors":0 158 }, 159 "busy":0.0, // from 0.0 (idle) to 1.0 (fully busy) 160 "writes":{ 161 "hz":0.0, 162 "counter":0, 163 "sectors":0 164 }, 165 "total_bytes":123412341234 // an estimate of total physical RAM 166 }, 167 "uptime_seconds":1234.2345, 168 "cpu":{ 169 "usage_cores":0.0 // average number of logical cores utilized by the process over the recent past; value may be > 1.0 170 }, 171 "network":{ 172 "current_connections":0, 173 "connections_established":{ 174 "hz":0.0 175 }, 176 "connections_closed":{ 177 "hz":0.0 178 }, 179 "connection_errors":{ 180 "hz":0.0 181 }, 182 "megabits_sent":{ 183 "hz":0.0 184 }, 185 "megabits_received":{ 186 "hz":0.0 187 } 188 } 189 } 190 }, 191 "old_logs":[ 192 { 193 "logs":[ // this field will be absent if a value has not been explicitly set 194 { 195 "id":"7f8d623d0cb9966e", 196 "healthy":true, 197 "address":"1.2.3.4:1234" 198 } 199 ], 200 "log_replication_factor":3, 201 "log_write_anti_quorum":0, 202 "log_fault_tolerance":2, 203 "remote_log_replication_factor":3, 204 "remote_log_fault_tolerance":2, 205 "satellite_log_replication_factor":3, 206 "satellite_log_write_anti_quorum":0, 207 "satellite_log_fault_tolerance":2 208 } 209 ], 210 "fault_tolerance":{ 211 "max_machine_failures_without_losing_availability":0, 212 "max_machine_failures_without_losing_data":0 213 }, 214 "qos":{ 215 "worst_queue_bytes_log_server":460, 216 "batch_performance_limited_by":{ 217 "reason_server_id":"7f8d623d0cb9966e", 218 "reason_id":0, 219 "name":{ // when not limiting 220 "$enum":[ 221 "workload", 222 "storage_server_write_queue_size", 223 "storage_server_write_bandwidth_mvcc", 224 "storage_server_readable_behind", 225 "log_server_mvcc_write_bandwidth", 226 "log_server_write_queue", 227 "storage_server_min_free_space", 228 "storage_server_min_free_space_ratio", 229 "log_server_min_free_space", 230 "log_server_min_free_space_ratio" 231 ] 232 }, 233 "description":"The database is not being saturated by the workload." 234 }, 235 "performance_limited_by":{ 236 "reason_server_id":"7f8d623d0cb9966e", 237 "reason_id":0, 238 "name":{ // when not limiting 239 "$enum":[ 240 "workload", 241 "storage_server_write_queue_size", 242 "storage_server_write_bandwidth_mvcc", 243 "storage_server_readable_behind", 244 "log_server_mvcc_write_bandwidth", 245 "log_server_write_queue", 246 "storage_server_min_free_space", 247 "storage_server_min_free_space_ratio", 248 "log_server_min_free_space", 249 "log_server_min_free_space_ratio" 250 ] 251 }, 252 "description":"The database is not being saturated by the workload." 253 }, 254 "batch_transactions_per_second_limit":0, 255 "transactions_per_second_limit":0, 256 "batch_released_transactions_per_second":0, 257 "released_transactions_per_second":0, 258 "limiting_queue_bytes_storage_server":0, 259 "worst_queue_bytes_storage_server":0, 260 "limiting_version_lag_storage_server":0, 261 "worst_version_lag_storage_server":0 262 }, 263 "incompatible_connections":[ 264 ], 265 "datacenter_version_difference":0, 266 "degraded_processes":0, 267 "database_available":true, 268 "database_locked":false, 269 "generation":2, 270 "latency_probe":{ // all measurements are based on running sample transactions 271 "read_seconds":7, // time to perform a single read 272 "immediate_priority_transaction_start_seconds":0.0, // time to start a sample transaction at system immediate priority 273 "batch_priority_transaction_start_seconds":0.0, // time to start a sample transaction at batch priority 274 "transaction_start_seconds":0.0, // time to start a sample transaction at normal priority 275 "commit_seconds":0.02 // time to commit a sample transaction 276 }, 277 "clients":{ 278 "count":1, 279 "supported_versions":[ 280 { 281 "client_version":"3.0.0", 282 "connected_clients":[ 283 { 284 "address":"127.0.0.1:9898", 285 "log_group":"default" 286 } 287 ], 288 "count" : 1, 289 "protocol_version" : "fdb00a400050001", 290 "source_version" : "9430e1127b4991cbc5ab2b17f41cfffa5de07e9d" 291 } 292 ] 293 }, 294 "messages":[ 295 { 296 "reasons":[ 297 { 298 "description":"Blah." 299 } 300 ], 301 "unreachable_processes":[ 302 { 303 "address":"1.2.3.4:1234" 304 } 305 ], 306 "name":{ // when not limiting 307 "$enum":[ 308 "unreachable_master_worker", 309 "unreachable_dataDistributor_worker", 310 "unreachable_ratekeeper_worker", 311 "unreadable_configuration", 312 "full_replication_timeout", 313 "client_issues", 314 "unreachable_processes", 315 "immediate_priority_transaction_start_probe_timeout", 316 "batch_priority_transaction_start_probe_timeout", 317 "transaction_start_probe_timeout", 318 "read_probe_timeout", 319 "commit_probe_timeout", 320 "storage_servers_error", 321 "status_incomplete", 322 "layer_status_incomplete", 323 "database_availability_timeout" 324 ] 325 }, 326 "issues":[ 327 { 328 "name":{ // when not limiting 329 "$enum":[ 330 "incorrect_cluster_file_contents" 331 ] 332 }, 333 "description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally." 334 } 335 ], 336 "description":"abc" 337 } 338 ], 339 "recovery_state":{ 340 "required_resolvers":1, 341 "required_proxies":1, 342 "name":{ // "fully_recovered" is the healthy state; other states are normal to transition through but not to persist in 343 "$enum":[ 344 "reading_coordinated_state", 345 "locking_coordinated_state", 346 "locking_old_transaction_servers", 347 "reading_transaction_system_state", 348 "configuration_missing", 349 "configuration_never_created", 350 "configuration_invalid", 351 "recruiting_transaction_servers", 352 "initializing_transaction_servers", 353 "recovery_transaction", 354 "writing_coordinated_state", 355 "accepting_commits", 356 "all_logs_recruited", 357 "storage_recovered", 358 "fully_recovered" 359 ] 360 }, 361 "required_logs":3, 362 "missing_logs":"7f8d623d0cb9966e", 363 "description":"Recovery complete." 364 }, 365 "workload":{ 366 "operations":{ 367 "writes":{ 368 "hz":0.0, 369 "counter":0, 370 "roughness":0.0 371 }, 372 "reads":{ 373 "hz":0.0, 374 "counter":0, 375 "roughness":0.0 376 }, 377 "read_requests":{ 378 "hz":0.0, 379 "counter":0, 380 "roughness":0.0 381 } 382 }, 383 "bytes":{ // of operations (independent of hz). Perfectly spaced operations will have a roughness of 1.0 . Randomly spaced (Poisson-distributed) operations will have a roughness of 2.0, with increased bunching resulting in increased values. Higher roughness can result in increased latency due to increased queuing. 384 "written":{ 385 "hz":0.0, 386 "counter":0, 387 "roughness":0.0 388 }, 389 "read":{ 390 "hz":0.0, 391 "counter":0, 392 "roughness":0.0 393 } 394 }, 395 "keys":{ 396 "read":{ 397 "hz":0.0, 398 "counter":0, 399 "roughness":0.0 400 } 401 }, 402 "transactions":{ 403 "started":{ 404 "hz":0.0, 405 "counter":0, 406 "roughness":0.0 407 }, 408 "conflicted":{ 409 "hz":0.0, 410 "counter":0, 411 "roughness":0.0 412 }, 413 "committed":{ 414 "hz":0.0, 415 "counter":0, 416 "roughness":0.0 417 } 418 } 419 }, 420 "cluster_controller_timestamp":1415650089, 421 "protocol_version":"fdb00a400050001", 422 "connection_string":"a:a@127.0.0.1:4000", 423 "full_replication":true, 424 "maintenance_zone":"0ccb4e0fdbdb5583010f6b77d9d10ece", 425 "maintenance_seconds_remaining":1.0, 426 "configuration":{ 427 "log_anti_quorum":0, 428 "log_replicas":2, 429 "log_replication_policy":"(zoneid^3x1)", 430 "redundancy_mode":{ 431 "$enum":[ 432 "single", 433 "double", 434 "triple", 435 "three_datacenter", 436 "three_datacenter_fallback", 437 "three_data_hall", 438 "three_data_hall_fallback" 439 ]}, 440 "regions":[{ 441 "datacenters":[{ 442 "id":"mr", 443 "priority":1, 444 "satellite":1 445 }], 446 "satellite_redundancy_mode":{ 447 "$enum":[ 448 "one_satellite_single", 449 "one_satellite_double", 450 "one_satellite_triple", 451 "two_satellite_safe", 452 "two_satellite_fast" 453 ]}, 454 "satellite_log_replicas":1, 455 "satellite_usable_dcs":1, 456 "satellite_anti_quorum":0, 457 "satellite_log_policy":"(zoneid^3x1)", 458 "satellite_logs":2 459 }], 460 "remote_redundancy_mode":{ 461 "$enum":[ 462 "remote_default", 463 "remote_single", 464 "remote_double", 465 "remote_triple", 466 "remote_three_data_hall" 467 ]}, 468 "remote_log_replicas":3, 469 "remote_logs":5, 470 "log_routers":10, 471 "usable_regions":1, 472 "repopulate_anti_quorum":1, 473 "storage_replicas":1, 474 "resolvers":1, // this field will be absent if a value has not been explicitly set 475 "storage_replication_policy":"(zoneid^3x1)", 476 "logs":2, // this field will be absent if a value has not been explicitly set 477 "log_version":2, 478 "log_engine":1, 479 "log_spill":1, 480 "storage_engine":{ 481 "$enum":[ 482 "ssd", 483 "ssd-1", 484 "ssd-2", 485 "ssd-redwood-experimental", 486 "memory" 487 ]}, 488 "coordinators_count":1, 489 "excluded_servers":[ 490 { 491 "address":"10.0.4.1" 492 } 493 ], 494 "auto_proxies":3, 495 "auto_resolvers":1, 496 "auto_logs":3, 497 "proxies":5 // this field will be absent if a value has not been explicitly set 498 }, 499 "data":{ 500 "least_operating_space_bytes_log_server":0, 501 "average_partition_size_bytes":0, 502 "state":{ 503 "healthy":true, 504 "min_replicas_remaining":0, 505 "name":{ // when not limiting 506 "$enum":[ 507 "initializing", 508 "missing_data", 509 "healing", 510 "optimizing_team_collections", 511 "healthy_repartitioning", 512 "healthy_removing_server", 513 "healthy_rebalancing", 514 "healthy" 515 ] 516 }, 517 "description":"" 518 }, 519 "least_operating_space_ratio_storage_server":0.1, 520 "max_machine_failures_without_losing_availability":0, 521 "total_disk_used_bytes":0, 522 "total_kv_size_bytes":0, // estimated 523 "partitions_count":2, 524 "moving_data":{ 525 "total_written_bytes":0, 526 "in_flight_bytes":0, 527 "in_queue_bytes":0, 528 "highest_priority":0 529 }, 530 "team_trackers":[ 531 { 532 "primary":true, 533 "in_flight_bytes":0, 534 "unhealthy_servers":0, 535 "state":{ 536 "healthy":true, 537 "min_replicas_remaining":0, 538 "name":{ // when not limiting 539 "$enum":[ 540 "initializing", 541 "missing_data", 542 "healing", 543 "optimizing_team_collections", 544 "healthy_repartitioning", 545 "healthy_removing_server", 546 "healthy_rebalancing", 547 "healthy" 548 ] 549 }, 550 "description":"" 551 } 552 } 553 ], 554 "least_operating_space_bytes_storage_server":0, 555 "max_machine_failures_without_losing_data":0 556 }, 557 "machines":{ 558 "$map_key=machineid":{ 559 "network":{ 560 "megabits_sent":{ 561 "hz":0.0 562 }, 563 "megabits_received":{ 564 "hz":0.0 565 }, 566 "tcp_segments_retransmitted":{ 567 "hz":0.0 568 } 569 }, 570 "memory":{ 571 "free_bytes":0, // an estimate of how many bytes are free to allocate to fdbservers without swapping 572 "committed_bytes":0, 573 "total_bytes":0 // an estimate of total physical RAM 574 }, 575 "contributing_workers":4, 576 "datacenter_id":"6344abf1813eb05b", 577 "excluded":false, 578 "address":"1.2.3.4", 579 "machine_id":"6344abf1813eb05b", 580 "locality":{ // This will contain any locality fields that are provided on the command line 581 "$map_key=localityName":"value" 582 }, 583 "cpu":{ 584 "logical_core_utilization":0.4 // computed as cpu_seconds / elapsed_seconds; value may be capped at 0.5 due to hyper-threading 585 } 586 } 587 } 588 }, 589 "client":{ 590 "coordinators":{ 591 "coordinators":[ 592 { 593 "reachable":true, 594 "address":"127.0.0.1:4701" 595 } 596 ], 597 "quorum_reachable":true 598 }, 599 "database_status":{ 600 "available":true, 601 "healthy":true 602 }, 603 "messages":[ 604 { 605 "name":{ // when not limiting 606 "$enum":[ 607 "inconsistent_cluster_file", 608 "unreachable_cluster_controller", 609 "no_cluster_controller", 610 "status_incomplete_client", 611 "status_incomplete_coordinators", 612 "status_incomplete_error", 613 "status_incomplete_timeout", 614 "status_incomplete_cluster", 615 "quorum_not_reachable" 616 ] 617 }, 618 "description":"The cluster file is not up to date." 619 } 620 ], 621 "timestamp":1415650089, 622 "cluster_file":{ 623 "path":"/etc/foundationdb/fdb.cluster", 624 "up_to_date":true 625 } 626 } 627