1package command
2
3import (
4	"fmt"
5	"math"
6	"sort"
7	"strconv"
8	"strings"
9	"time"
10
11	humanize "github.com/dustin/go-humanize"
12	"github.com/hashicorp/nomad/api"
13	"github.com/hashicorp/nomad/api/contexts"
14	"github.com/hashicorp/nomad/helper"
15	"github.com/posener/complete"
16)
17
18const (
19	// floatFormat is a format string for formatting floats.
20	floatFormat = "#,###.##"
21
22	// bytesPerMegabyte is the number of bytes per MB
23	bytesPerMegabyte = 1024 * 1024
24)
25
26type NodeStatusCommand struct {
27	Meta
28	length      int
29	short       bool
30	verbose     bool
31	list_allocs bool
32	self        bool
33	stats       bool
34	json        bool
35	tmpl        string
36}
37
38func (c *NodeStatusCommand) Help() string {
39	helpText := `
40Usage: nomad node status [options] <node>
41
42  Display status information about a given node. The list of nodes
43  returned includes only nodes which jobs may be scheduled to, and
44  includes status and other high-level information.
45
46  If a node ID is passed, information for that specific node will be displayed,
47  including resource usage statistics. If no node ID's are passed, then a
48  short-hand list of all nodes will be displayed. The -self flag is useful to
49  quickly access the status of the local node.
50
51General Options:
52
53  ` + generalOptionsUsage() + `
54
55Node Status Options:
56
57  -self
58    Query the status of the local node.
59
60  -stats
61    Display detailed resource usage statistics.
62
63  -allocs
64    Display a count of running allocations for each node.
65
66  -short
67    Display short output. Used only when a single node is being
68    queried, and drops verbose output about node allocations.
69
70  -verbose
71    Display full information.
72
73  -json
74    Output the node in its JSON format.
75
76  -t
77    Format and display node using a Go template.
78`
79	return strings.TrimSpace(helpText)
80}
81
82func (c *NodeStatusCommand) Synopsis() string {
83	return "Display status information about nodes"
84}
85
86func (c *NodeStatusCommand) AutocompleteFlags() complete.Flags {
87	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
88		complete.Flags{
89			"-allocs":  complete.PredictNothing,
90			"-json":    complete.PredictNothing,
91			"-self":    complete.PredictNothing,
92			"-short":   complete.PredictNothing,
93			"-stats":   complete.PredictNothing,
94			"-t":       complete.PredictAnything,
95			"-verbose": complete.PredictNothing,
96		})
97}
98
99func (c *NodeStatusCommand) AutocompleteArgs() complete.Predictor {
100	return complete.PredictFunc(func(a complete.Args) []string {
101		client, err := c.Meta.Client()
102		if err != nil {
103			return nil
104		}
105
106		resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Nodes, nil)
107		if err != nil {
108			return []string{}
109		}
110		return resp.Matches[contexts.Nodes]
111	})
112}
113
114func (c *NodeStatusCommand) Name() string { return "node-status" }
115
116func (c *NodeStatusCommand) Run(args []string) int {
117
118	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
119	flags.Usage = func() { c.Ui.Output(c.Help()) }
120	flags.BoolVar(&c.short, "short", false, "")
121	flags.BoolVar(&c.verbose, "verbose", false, "")
122	flags.BoolVar(&c.list_allocs, "allocs", false, "")
123	flags.BoolVar(&c.self, "self", false, "")
124	flags.BoolVar(&c.stats, "stats", false, "")
125	flags.BoolVar(&c.json, "json", false, "")
126	flags.StringVar(&c.tmpl, "t", "", "")
127
128	if err := flags.Parse(args); err != nil {
129		return 1
130	}
131
132	// Check that we got either a single node or none
133	args = flags.Args()
134	if len(args) > 1 {
135		c.Ui.Error("This command takes either one or no arguments")
136		c.Ui.Error(commandErrorText(c))
137		return 1
138	}
139
140	// Truncate the id unless full length is requested
141	c.length = shortId
142	if c.verbose {
143		c.length = fullId
144	}
145
146	// Get the HTTP client
147	client, err := c.Meta.Client()
148	if err != nil {
149		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
150		return 1
151	}
152
153	// Use list mode if no node name was provided
154	if len(args) == 0 && !c.self {
155
156		// Query the node info
157		nodes, _, err := client.Nodes().List(nil)
158		if err != nil {
159			c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err))
160			return 1
161		}
162
163		// If output format is specified, format and output the node data list
164		if c.json || len(c.tmpl) > 0 {
165			out, err := Format(c.json, c.tmpl, nodes)
166			if err != nil {
167				c.Ui.Error(err.Error())
168				return 1
169			}
170
171			c.Ui.Output(out)
172			return 0
173		}
174
175		// Return nothing if no nodes found
176		if len(nodes) == 0 {
177			return 0
178		}
179
180		// Format the nodes list
181		out := make([]string, len(nodes)+1)
182
183		out[0] = "ID|DC|Name|Class|"
184
185		if c.verbose {
186			out[0] += "Address|Version|"
187		}
188
189		out[0] += "Drain|Eligibility|Status"
190
191		if c.list_allocs {
192			out[0] += "|Running Allocs"
193		}
194
195		for i, node := range nodes {
196			out[i+1] = fmt.Sprintf("%s|%s|%s|%s",
197				limit(node.ID, c.length),
198				node.Datacenter,
199				node.Name,
200				node.NodeClass)
201			if c.verbose {
202				out[i+1] += fmt.Sprintf("|%s|%s",
203					node.Address, node.Version)
204			}
205			out[i+1] += fmt.Sprintf("|%v|%s|%s",
206				node.Drain,
207				node.SchedulingEligibility,
208				node.Status)
209
210			if c.list_allocs {
211				numAllocs, err := getRunningAllocs(client, node.ID)
212				if err != nil {
213					c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err))
214					return 1
215				}
216				out[i+1] += fmt.Sprintf("|%v",
217					len(numAllocs))
218			}
219		}
220
221		// Dump the output
222		c.Ui.Output(formatList(out))
223		return 0
224	}
225
226	// Query the specific node
227	var nodeID string
228	if !c.self {
229		nodeID = args[0]
230	} else {
231		var err error
232		if nodeID, err = getLocalNodeID(client); err != nil {
233			c.Ui.Error(err.Error())
234			return 1
235		}
236	}
237	if len(nodeID) == 1 {
238		c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters."))
239		return 1
240	}
241
242	nodeID = sanitizeUUIDPrefix(nodeID)
243	nodes, _, err := client.Nodes().PrefixList(nodeID)
244	if err != nil {
245		c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err))
246		return 1
247	}
248	// Return error if no nodes are found
249	if len(nodes) == 0 {
250		c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID))
251		return 1
252	}
253	if len(nodes) > 1 {
254		// Dump the output
255		c.Ui.Error(fmt.Sprintf("Prefix matched multiple nodes\n\n%s",
256			formatNodeStubList(nodes, c.verbose)))
257		return 1
258	}
259
260	// Prefix lookup matched a single node
261	node, _, err := client.Nodes().Info(nodes[0].ID, nil)
262	if err != nil {
263		c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err))
264		return 1
265	}
266
267	// If output format is specified, format and output the data
268	if c.json || len(c.tmpl) > 0 {
269		out, err := Format(c.json, c.tmpl, node)
270		if err != nil {
271			c.Ui.Error(err.Error())
272			return 1
273		}
274
275		c.Ui.Output(out)
276		return 0
277	}
278
279	return c.formatNode(client, node)
280}
281
282func nodeDrivers(n *api.Node) []string {
283	var drivers []string
284	for k, v := range n.Attributes {
285		// driver.docker = 1
286		parts := strings.Split(k, ".")
287		if len(parts) != 2 {
288			continue
289		} else if parts[0] != "driver" {
290			continue
291		} else if v != "1" {
292			continue
293		}
294
295		drivers = append(drivers, parts[1])
296	}
297
298	sort.Strings(drivers)
299	return drivers
300}
301
302func nodeVolumeNames(n *api.Node) []string {
303	var volumes []string
304	for name := range n.HostVolumes {
305		volumes = append(volumes, name)
306	}
307
308	sort.Strings(volumes)
309	return volumes
310}
311
312func formatDrain(n *api.Node) string {
313	if n.DrainStrategy != nil {
314		b := new(strings.Builder)
315		b.WriteString("true")
316		if n.DrainStrategy.DrainSpec.Deadline.Nanoseconds() < 0 {
317			b.WriteString("; force drain")
318		} else if n.DrainStrategy.ForceDeadline.IsZero() {
319			b.WriteString("; no deadline")
320		} else {
321			fmt.Fprintf(b, "; %s deadline", formatTime(n.DrainStrategy.ForceDeadline))
322		}
323
324		if n.DrainStrategy.IgnoreSystemJobs {
325			b.WriteString("; ignoring system jobs")
326		}
327		return b.String()
328	}
329
330	return strconv.FormatBool(n.Drain)
331}
332
333func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int {
334	// Format the header output
335	basic := []string{
336		fmt.Sprintf("ID|%s", node.ID),
337		fmt.Sprintf("Name|%s", node.Name),
338		fmt.Sprintf("Class|%s", node.NodeClass),
339		fmt.Sprintf("DC|%s", node.Datacenter),
340		fmt.Sprintf("Drain|%v", formatDrain(node)),
341		fmt.Sprintf("Eligibility|%s", node.SchedulingEligibility),
342		fmt.Sprintf("Status|%s", node.Status),
343	}
344
345	if c.short {
346		basic = append(basic, fmt.Sprintf("Host Volumes|%s", strings.Join(nodeVolumeNames(node), ",")))
347		basic = append(basic, fmt.Sprintf("Drivers|%s", strings.Join(nodeDrivers(node), ",")))
348		c.Ui.Output(c.Colorize().Color(formatKV(basic)))
349
350		// Output alloc info
351		if err := c.outputAllocInfo(client, node); err != nil {
352			c.Ui.Error(fmt.Sprintf("%s", err))
353			return 1
354		}
355
356		return 0
357	}
358
359	// Get the host stats
360	hostStats, nodeStatsErr := client.Nodes().Stats(node.ID, nil)
361	if nodeStatsErr != nil {
362		c.Ui.Output("")
363		c.Ui.Error(fmt.Sprintf("error fetching node stats: %v", nodeStatsErr))
364	}
365	if hostStats != nil {
366		uptime := time.Duration(hostStats.Uptime * uint64(time.Second))
367		basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String()))
368	}
369
370	// When we're not running in verbose mode, then also include host volumes and
371	// driver info in the basic output
372	if !c.verbose {
373		basic = append(basic, fmt.Sprintf("Host Volumes|%s", strings.Join(nodeVolumeNames(node), ",")))
374
375		driverStatus := fmt.Sprintf("Driver Status| %s", c.outputTruncatedNodeDriverInfo(node))
376		basic = append(basic, driverStatus)
377	}
378
379	// Output the basic info
380	c.Ui.Output(c.Colorize().Color(formatKV(basic)))
381
382	// If we're running in verbose mode, include full host volume and driver info
383	if c.verbose {
384		c.outputNodeVolumeInfo(node)
385		c.outputNodeDriverInfo(node)
386	}
387
388	// Emit node events
389	c.outputNodeStatusEvents(node)
390
391	// Get list of running allocations on the node
392	runningAllocs, err := getRunningAllocs(client, node.ID)
393	if err != nil {
394		c.Ui.Error(fmt.Sprintf("Error querying node for running allocations: %s", err))
395		return 1
396	}
397
398	allocatedResources := getAllocatedResources(client, runningAllocs, node)
399	c.Ui.Output(c.Colorize().Color("\n[bold]Allocated Resources[reset]"))
400	c.Ui.Output(formatList(allocatedResources))
401
402	actualResources, err := getActualResources(client, runningAllocs, node)
403	if err == nil {
404		c.Ui.Output(c.Colorize().Color("\n[bold]Allocation Resource Utilization[reset]"))
405		c.Ui.Output(formatList(actualResources))
406	}
407
408	hostResources, err := getHostResources(hostStats, node)
409	if err != nil {
410		c.Ui.Output("")
411		c.Ui.Error(fmt.Sprintf("error fetching node stats: %v", err))
412	}
413	if err == nil {
414		c.Ui.Output(c.Colorize().Color("\n[bold]Host Resource Utilization[reset]"))
415		c.Ui.Output(formatList(hostResources))
416	}
417
418	if err == nil && node.NodeResources != nil && len(node.NodeResources.Devices) > 0 {
419		c.Ui.Output(c.Colorize().Color("\n[bold]Device Resource Utilization[reset]"))
420		c.Ui.Output(formatList(getDeviceResourcesForNode(hostStats.DeviceStats, node)))
421	}
422	if hostStats != nil && c.stats {
423		c.Ui.Output(c.Colorize().Color("\n[bold]CPU Stats[reset]"))
424		c.printCpuStats(hostStats)
425		c.Ui.Output(c.Colorize().Color("\n[bold]Memory Stats[reset]"))
426		c.printMemoryStats(hostStats)
427		c.Ui.Output(c.Colorize().Color("\n[bold]Disk Stats[reset]"))
428		c.printDiskStats(hostStats)
429		if len(hostStats.DeviceStats) > 0 {
430			c.Ui.Output(c.Colorize().Color("\n[bold]Device Stats[reset]"))
431			printDeviceStats(c.Ui, hostStats.DeviceStats)
432		}
433	}
434
435	if err := c.outputAllocInfo(client, node); err != nil {
436		c.Ui.Error(fmt.Sprintf("%s", err))
437		return 1
438	}
439
440	return 0
441}
442
443func (c *NodeStatusCommand) outputAllocInfo(client *api.Client, node *api.Node) error {
444	nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil)
445	if err != nil {
446		return fmt.Errorf("Error querying node allocations: %s", err)
447	}
448
449	c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]"))
450	c.Ui.Output(formatAllocList(nodeAllocs, c.verbose, c.length))
451
452	if c.verbose {
453		c.formatAttributes(node)
454		c.formatDeviceAttributes(node)
455		c.formatMeta(node)
456	}
457
458	return nil
459}
460
461func (c *NodeStatusCommand) outputTruncatedNodeDriverInfo(node *api.Node) string {
462	drivers := make([]string, 0, len(node.Drivers))
463
464	for driverName, driverInfo := range node.Drivers {
465		if !driverInfo.Detected {
466			continue
467		}
468
469		if !driverInfo.Healthy {
470			drivers = append(drivers, fmt.Sprintf("%s (unhealthy)", driverName))
471		} else {
472			drivers = append(drivers, driverName)
473		}
474	}
475	sort.Strings(drivers)
476	return strings.Trim(strings.Join(drivers, ","), ", ")
477}
478
479func (c *NodeStatusCommand) outputNodeVolumeInfo(node *api.Node) {
480	c.Ui.Output(c.Colorize().Color("\n[bold]Host Volumes"))
481
482	names := make([]string, 0, len(node.HostVolumes))
483	for name := range node.HostVolumes {
484		names = append(names, name)
485	}
486	sort.Strings(names)
487
488	output := make([]string, 0, len(names)+1)
489	output = append(output, "Name|ReadOnly|Source")
490
491	for _, volName := range names {
492		info := node.HostVolumes[volName]
493		output = append(output, fmt.Sprintf("%s|%v|%s", volName, info.ReadOnly, info.Path))
494	}
495	c.Ui.Output(formatList(output))
496}
497
498func (c *NodeStatusCommand) outputNodeDriverInfo(node *api.Node) {
499	c.Ui.Output(c.Colorize().Color("\n[bold]Drivers"))
500
501	size := len(node.Drivers)
502	nodeDrivers := make([]string, 0, size+1)
503
504	nodeDrivers = append(nodeDrivers, "Driver|Detected|Healthy|Message|Time")
505
506	drivers := make([]string, 0, len(node.Drivers))
507	for driver := range node.Drivers {
508		drivers = append(drivers, driver)
509	}
510	sort.Strings(drivers)
511
512	for _, driver := range drivers {
513		info := node.Drivers[driver]
514		timestamp := formatTime(info.UpdateTime)
515		nodeDrivers = append(nodeDrivers, fmt.Sprintf("%s|%v|%v|%s|%s", driver, info.Detected, info.Healthy, info.HealthDescription, timestamp))
516	}
517	c.Ui.Output(formatList(nodeDrivers))
518}
519
520func (c *NodeStatusCommand) outputNodeStatusEvents(node *api.Node) {
521	c.Ui.Output(c.Colorize().Color("\n[bold]Node Events"))
522	c.outputNodeEvent(node.Events)
523}
524
525func (c *NodeStatusCommand) outputNodeEvent(events []*api.NodeEvent) {
526	size := len(events)
527	nodeEvents := make([]string, size+1)
528	if c.verbose {
529		nodeEvents[0] = "Time|Subsystem|Message|Details"
530	} else {
531		nodeEvents[0] = "Time|Subsystem|Message"
532	}
533
534	for i, event := range events {
535		timestamp := formatTime(event.Timestamp)
536		subsystem := formatEventSubsystem(event.Subsystem, event.Details["driver"])
537		msg := event.Message
538		if c.verbose {
539			details := formatEventDetails(event.Details)
540			nodeEvents[size-i] = fmt.Sprintf("%s|%s|%s|%s", timestamp, subsystem, msg, details)
541		} else {
542			nodeEvents[size-i] = fmt.Sprintf("%s|%s|%s", timestamp, subsystem, msg)
543		}
544	}
545	c.Ui.Output(formatList(nodeEvents))
546}
547
548func formatEventSubsystem(subsystem, driverName string) string {
549	if driverName == "" {
550		return subsystem
551	}
552
553	// If this event is for a driver, append the driver name to make the message
554	// clearer
555	return fmt.Sprintf("Driver: %s", driverName)
556}
557
558func formatEventDetails(details map[string]string) string {
559	output := make([]string, 0, len(details))
560	for k, v := range details {
561		output = append(output, fmt.Sprintf("%s: %s ", k, v))
562	}
563	return strings.Join(output, ", ")
564}
565
566func (c *NodeStatusCommand) formatAttributes(node *api.Node) {
567	// Print the attributes
568	keys := make([]string, len(node.Attributes))
569	for k := range node.Attributes {
570		keys = append(keys, k)
571	}
572	sort.Strings(keys)
573
574	var attributes []string
575	for _, k := range keys {
576		if k != "" {
577			attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k]))
578		}
579	}
580	c.Ui.Output(c.Colorize().Color("\n[bold]Attributes[reset]"))
581	c.Ui.Output(formatKV(attributes))
582}
583
584func (c *NodeStatusCommand) formatDeviceAttributes(node *api.Node) {
585	if node.NodeResources == nil {
586		return
587	}
588	devices := node.NodeResources.Devices
589	if len(devices) == 0 {
590		return
591	}
592
593	sort.Slice(devices, func(i, j int) bool {
594		return devices[i].ID() < devices[j].ID()
595	})
596
597	first := true
598	for _, d := range devices {
599		if len(d.Attributes) == 0 {
600			continue
601		}
602
603		if first {
604			c.Ui.Output("\nDevice Group Attributes")
605			first = false
606		} else {
607			c.Ui.Output("")
608		}
609		c.Ui.Output(formatKV(getDeviceAttributes(d)))
610	}
611}
612
613func (c *NodeStatusCommand) formatMeta(node *api.Node) {
614	// Print the meta
615	keys := make([]string, 0, len(node.Meta))
616	for k := range node.Meta {
617		keys = append(keys, k)
618	}
619	sort.Strings(keys)
620
621	var meta []string
622	for _, k := range keys {
623		if k != "" {
624			meta = append(meta, fmt.Sprintf("%s|%s", k, node.Meta[k]))
625		}
626	}
627	c.Ui.Output(c.Colorize().Color("\n[bold]Meta[reset]"))
628	c.Ui.Output(formatKV(meta))
629}
630
631func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) {
632	l := len(hostStats.CPU)
633	for i, cpuStat := range hostStats.CPU {
634		cpuStatsAttr := make([]string, 4)
635		cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU)
636		cpuStatsAttr[1] = fmt.Sprintf("User|%v%%", humanize.FormatFloat(floatFormat, cpuStat.User))
637		cpuStatsAttr[2] = fmt.Sprintf("System|%v%%", humanize.FormatFloat(floatFormat, cpuStat.System))
638		cpuStatsAttr[3] = fmt.Sprintf("Idle|%v%%", humanize.FormatFloat(floatFormat, cpuStat.Idle))
639		c.Ui.Output(formatKV(cpuStatsAttr))
640		if i+1 < l {
641			c.Ui.Output("")
642		}
643	}
644}
645
646func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) {
647	memoryStat := hostStats.Memory
648	memStatsAttr := make([]string, 4)
649	memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.IBytes(memoryStat.Total))
650	memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.IBytes(memoryStat.Available))
651	memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.IBytes(memoryStat.Used))
652	memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.IBytes(memoryStat.Free))
653	c.Ui.Output(formatKV(memStatsAttr))
654}
655
656func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) {
657	l := len(hostStats.DiskStats)
658	for i, diskStat := range hostStats.DiskStats {
659		diskStatsAttr := make([]string, 7)
660		diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device)
661		diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint)
662		diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.IBytes(diskStat.Size))
663		diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.IBytes(diskStat.Used))
664		diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.IBytes(diskStat.Available))
665		diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.UsedPercent))
666		diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.InodesUsedPercent))
667		c.Ui.Output(formatKV(diskStatsAttr))
668		if i+1 < l {
669			c.Ui.Output("")
670		}
671	}
672}
673
674// getRunningAllocs returns a slice of allocation id's running on the node
675func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) {
676	var allocs []*api.Allocation
677
678	// Query the node allocations
679	nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil)
680	// Filter list to only running allocations
681	for _, alloc := range nodeAllocs {
682		if alloc.ClientStatus == "running" {
683			allocs = append(allocs, alloc)
684		}
685	}
686	return allocs, err
687}
688
689// getAllocatedResources returns the resource usage of the node.
690func getAllocatedResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) []string {
691	// Compute the total
692	total := computeNodeTotalResources(node)
693
694	// Get Resources
695	var cpu, mem, disk int
696	for _, alloc := range runningAllocs {
697		cpu += *alloc.Resources.CPU
698		mem += *alloc.Resources.MemoryMB
699		disk += *alloc.Resources.DiskMB
700	}
701
702	resources := make([]string, 2)
703	resources[0] = "CPU|Memory|Disk"
704	resources[1] = fmt.Sprintf("%d/%d MHz|%s/%s|%s/%s",
705		cpu,
706		*total.CPU,
707		humanize.IBytes(uint64(mem*bytesPerMegabyte)),
708		humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte)),
709		humanize.IBytes(uint64(disk*bytesPerMegabyte)),
710		humanize.IBytes(uint64(*total.DiskMB*bytesPerMegabyte)))
711
712	return resources
713}
714
715// computeNodeTotalResources returns the total allocatable resources (resources
716// minus reserved)
717func computeNodeTotalResources(node *api.Node) api.Resources {
718	total := api.Resources{}
719
720	r := node.Resources
721	res := node.Reserved
722	if res == nil {
723		res = &api.Resources{}
724	}
725	total.CPU = helper.IntToPtr(*r.CPU - *res.CPU)
726	total.MemoryMB = helper.IntToPtr(*r.MemoryMB - *res.MemoryMB)
727	total.DiskMB = helper.IntToPtr(*r.DiskMB - *res.DiskMB)
728	return total
729}
730
731// getActualResources returns the actual resource usage of the allocations.
732func getActualResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) ([]string, error) {
733	// Compute the total
734	total := computeNodeTotalResources(node)
735
736	// Get Resources
737	var cpu float64
738	var mem uint64
739	for _, alloc := range runningAllocs {
740		// Make the call to the client to get the actual usage.
741		stats, err := client.Allocations().Stats(alloc, nil)
742		if err != nil {
743			return nil, err
744		}
745
746		cpu += stats.ResourceUsage.CpuStats.TotalTicks
747		mem += stats.ResourceUsage.MemoryStats.RSS
748	}
749
750	resources := make([]string, 2)
751	resources[0] = "CPU|Memory"
752	resources[1] = fmt.Sprintf("%v/%d MHz|%v/%v",
753		math.Floor(cpu),
754		*total.CPU,
755		humanize.IBytes(mem),
756		humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte)))
757
758	return resources, nil
759}
760
761// getHostResources returns the actual resource usage of the node.
762func getHostResources(hostStats *api.HostStats, node *api.Node) ([]string, error) {
763	if hostStats == nil {
764		return nil, fmt.Errorf("actual resource usage not present")
765	}
766	var resources []string
767
768	// calculate disk usage
769	storageDevice := node.Attributes["unique.storage.volume"]
770	var diskUsed, diskSize uint64
771	var physical bool
772	for _, disk := range hostStats.DiskStats {
773		if disk.Device == storageDevice {
774			diskUsed = disk.Used
775			diskSize = disk.Size
776			physical = true
777		}
778	}
779
780	resources = make([]string, 2)
781	resources[0] = "CPU|Memory|Disk"
782	if physical {
783		resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|%s/%s",
784			math.Floor(hostStats.CPUTicksConsumed),
785			*node.Resources.CPU,
786			humanize.IBytes(hostStats.Memory.Used),
787			humanize.IBytes(hostStats.Memory.Total),
788			humanize.IBytes(diskUsed),
789			humanize.IBytes(diskSize),
790		)
791	} else {
792		// If non-physical device are used, output device name only,
793		// since nomad doesn't collect the stats data.
794		resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|(%s)",
795			math.Floor(hostStats.CPUTicksConsumed),
796			*node.Resources.CPU,
797			humanize.IBytes(hostStats.Memory.Used),
798			humanize.IBytes(hostStats.Memory.Total),
799			storageDevice,
800		)
801	}
802	return resources, nil
803}
804
805// formatNodeStubList is used to return a table format of a list of node stubs.
806func formatNodeStubList(nodes []*api.NodeListStub, verbose bool) string {
807	// Return error if no nodes are found
808	if len(nodes) == 0 {
809		return ""
810	}
811	// Truncate the id unless full length is requested
812	length := shortId
813	if verbose {
814		length = fullId
815	}
816
817	// Format the nodes list that matches the prefix so that the user
818	// can create a more specific request
819	out := make([]string, len(nodes)+1)
820	out[0] = "ID|DC|Name|Class|Drain|Eligibility|Status"
821	for i, node := range nodes {
822		out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s|%s",
823			limit(node.ID, length),
824			node.Datacenter,
825			node.Name,
826			node.NodeClass,
827			node.Drain,
828			node.SchedulingEligibility,
829			node.Status)
830	}
831
832	return formatList(out)
833}
834