1package command
2
3import (
4	"fmt"
5	"math"
6	"sort"
7	"strconv"
8	"strings"
9	"time"
10
11	humanize "github.com/dustin/go-humanize"
12	"github.com/hashicorp/nomad/api"
13	"github.com/hashicorp/nomad/api/contexts"
14	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/restarts"
15	"github.com/posener/complete"
16)
17
18type AllocStatusCommand struct {
19	Meta
20}
21
22func (c *AllocStatusCommand) Help() string {
23	helpText := `
24Usage: nomad alloc status [options] <allocation>
25
26  Display information about existing allocations and its tasks. This command can
27  be used to inspect the current status of an allocation, including its running
28  status, metadata, and verbose failure messages reported by internal
29  subsystems.
30
31General Options:
32
33  ` + generalOptionsUsage() + `
34
35Alloc Status Options:
36
37  -short
38    Display short output. Shows only the most recent task event.
39
40  -stats
41    Display detailed resource usage statistics.
42
43  -verbose
44    Show full information.
45
46  -json
47    Output the allocation in its JSON format.
48
49  -t
50    Format and display allocation using a Go template.
51`
52
53	return strings.TrimSpace(helpText)
54}
55
56func (c *AllocStatusCommand) Synopsis() string {
57	return "Display allocation status information and metadata"
58}
59
60func (c *AllocStatusCommand) AutocompleteFlags() complete.Flags {
61	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
62		complete.Flags{
63			"-short":   complete.PredictNothing,
64			"-verbose": complete.PredictNothing,
65			"-json":    complete.PredictNothing,
66			"-t":       complete.PredictAnything,
67		})
68}
69
70func (c *AllocStatusCommand) AutocompleteArgs() complete.Predictor {
71	return complete.PredictFunc(func(a complete.Args) []string {
72		client, err := c.Meta.Client()
73		if err != nil {
74			return nil
75		}
76
77		resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Allocs, nil)
78		if err != nil {
79			return []string{}
80		}
81		return resp.Matches[contexts.Allocs]
82	})
83}
84
85func (c *AllocStatusCommand) Name() string { return "alloc status" }
86
87func (c *AllocStatusCommand) Run(args []string) int {
88	var short, displayStats, verbose, json bool
89	var tmpl string
90
91	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
92	flags.Usage = func() { c.Ui.Output(c.Help()) }
93	flags.BoolVar(&short, "short", false, "")
94	flags.BoolVar(&verbose, "verbose", false, "")
95	flags.BoolVar(&displayStats, "stats", false, "")
96	flags.BoolVar(&json, "json", false, "")
97	flags.StringVar(&tmpl, "t", "", "")
98
99	if err := flags.Parse(args); err != nil {
100		return 1
101	}
102
103	// Check that we got exactly one allocation ID
104	args = flags.Args()
105
106	// Get the HTTP client
107	client, err := c.Meta.Client()
108	if err != nil {
109		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
110		return 1
111	}
112
113	// If args not specified but output format is specified, format and output the allocations data list
114	if len(args) == 0 && json || len(tmpl) > 0 {
115		allocs, _, err := client.Allocations().List(nil)
116		if err != nil {
117			c.Ui.Error(fmt.Sprintf("Error querying allocations: %v", err))
118			return 1
119		}
120
121		out, err := Format(json, tmpl, allocs)
122		if err != nil {
123			c.Ui.Error(err.Error())
124			return 1
125		}
126
127		c.Ui.Output(out)
128		return 0
129	}
130
131	if len(args) != 1 {
132		c.Ui.Error("This command takes one of the following argument conditions:")
133		c.Ui.Error(" * A single <allocation>")
134		c.Ui.Error(" * No arguments, with output format specified")
135		c.Ui.Error(commandErrorText(c))
136		return 1
137	}
138	allocID := args[0]
139
140	// Truncate the id unless full length is requested
141	length := shortId
142	if verbose {
143		length = fullId
144	}
145
146	// Query the allocation info
147	if len(allocID) == 1 {
148		c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters."))
149		return 1
150	}
151
152	allocID = sanitizeUUIDPrefix(allocID)
153	allocs, _, err := client.Allocations().PrefixList(allocID)
154	if err != nil {
155		c.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err))
156		return 1
157	}
158	if len(allocs) == 0 {
159		c.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID))
160		return 1
161	}
162	if len(allocs) > 1 {
163		out := formatAllocListStubs(allocs, verbose, length)
164		c.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", out))
165		return 0
166	}
167	// Prefix lookup matched a single allocation
168	alloc, _, err := client.Allocations().Info(allocs[0].ID, nil)
169	if err != nil {
170		c.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err))
171		return 1
172	}
173
174	// If output format is specified, format and output the data
175	if json || len(tmpl) > 0 {
176		out, err := Format(json, tmpl, alloc)
177		if err != nil {
178			c.Ui.Error(err.Error())
179			return 1
180		}
181
182		c.Ui.Output(out)
183		return 0
184	}
185
186	// Format the allocation data
187	output, err := formatAllocBasicInfo(alloc, client, length, verbose)
188	if err != nil {
189		c.Ui.Error(err.Error())
190		return 1
191	}
192	c.Ui.Output(output)
193
194	if len(alloc.AllocatedResources.Shared.Networks) > 0 && alloc.AllocatedResources.Shared.Networks[0].HasPorts() {
195		c.Ui.Output("")
196		c.Ui.Output(formatAllocNetworkInfo(alloc))
197	}
198
199	if short {
200		c.shortTaskStatus(alloc)
201	} else {
202		var statsErr error
203		var stats *api.AllocResourceUsage
204		stats, statsErr = client.Allocations().Stats(alloc, nil)
205		if statsErr != nil {
206			c.Ui.Output("")
207			if statsErr != api.NodeDownErr {
208				c.Ui.Error(fmt.Sprintf("Couldn't retrieve stats: %v", statsErr))
209			} else {
210				c.Ui.Output("Omitting resource statistics since the node is down.")
211			}
212		}
213		c.outputTaskDetails(alloc, stats, displayStats)
214	}
215
216	// Format the detailed status
217	if verbose {
218		c.Ui.Output(c.Colorize().Color("\n[bold]Placement Metrics[reset]"))
219		c.Ui.Output(formatAllocMetrics(alloc.Metrics, true, "  "))
220	}
221
222	return 0
223}
224
225func formatAllocBasicInfo(alloc *api.Allocation, client *api.Client, uuidLength int, verbose bool) (string, error) {
226	var formattedCreateTime, formattedModifyTime string
227
228	if verbose {
229		formattedCreateTime = formatUnixNanoTime(alloc.CreateTime)
230		formattedModifyTime = formatUnixNanoTime(alloc.ModifyTime)
231	} else {
232		formattedCreateTime = prettyTimeDiff(time.Unix(0, alloc.CreateTime), time.Now())
233		formattedModifyTime = prettyTimeDiff(time.Unix(0, alloc.ModifyTime), time.Now())
234	}
235
236	basic := []string{
237		fmt.Sprintf("ID|%s", alloc.ID),
238		fmt.Sprintf("Eval ID|%s", limit(alloc.EvalID, uuidLength)),
239		fmt.Sprintf("Name|%s", alloc.Name),
240		fmt.Sprintf("Node ID|%s", limit(alloc.NodeID, uuidLength)),
241		fmt.Sprintf("Node Name|%s", alloc.NodeName),
242		fmt.Sprintf("Job ID|%s", alloc.JobID),
243		fmt.Sprintf("Job Version|%d", *alloc.Job.Version),
244		fmt.Sprintf("Client Status|%s", alloc.ClientStatus),
245		fmt.Sprintf("Client Description|%s", alloc.ClientDescription),
246		fmt.Sprintf("Desired Status|%s", alloc.DesiredStatus),
247		fmt.Sprintf("Desired Description|%s", alloc.DesiredDescription),
248		fmt.Sprintf("Created|%s", formattedCreateTime),
249		fmt.Sprintf("Modified|%s", formattedModifyTime),
250	}
251
252	if alloc.DeploymentID != "" {
253		health := "unset"
254		canary := false
255		if alloc.DeploymentStatus != nil {
256			if alloc.DeploymentStatus.Healthy != nil {
257				if *alloc.DeploymentStatus.Healthy {
258					health = "healthy"
259				} else {
260					health = "unhealthy"
261				}
262			}
263
264			canary = alloc.DeploymentStatus.Canary
265		}
266
267		basic = append(basic,
268			fmt.Sprintf("Deployment ID|%s", limit(alloc.DeploymentID, uuidLength)),
269			fmt.Sprintf("Deployment Health|%s", health))
270		if canary {
271			basic = append(basic, fmt.Sprintf("Canary|%v", true))
272		}
273	}
274
275	if alloc.RescheduleTracker != nil && len(alloc.RescheduleTracker.Events) > 0 {
276		attempts, total := alloc.RescheduleInfo(time.Unix(0, alloc.ModifyTime))
277		// Show this section only if the reschedule policy limits the number of attempts
278		if total > 0 {
279			reschedInfo := fmt.Sprintf("Reschedule Attempts|%d/%d", attempts, total)
280			basic = append(basic, reschedInfo)
281		}
282	}
283	if alloc.NextAllocation != "" {
284		basic = append(basic,
285			fmt.Sprintf("Replacement Alloc ID|%s", limit(alloc.NextAllocation, uuidLength)))
286	}
287	if alloc.FollowupEvalID != "" {
288		nextEvalTime := futureEvalTimePretty(alloc.FollowupEvalID, client)
289		if nextEvalTime != "" {
290			basic = append(basic,
291				fmt.Sprintf("Reschedule Eligibility|%s", nextEvalTime))
292		}
293	}
294
295	if verbose {
296		basic = append(basic,
297			fmt.Sprintf("Evaluated Nodes|%d", alloc.Metrics.NodesEvaluated),
298			fmt.Sprintf("Filtered Nodes|%d", alloc.Metrics.NodesFiltered),
299			fmt.Sprintf("Exhausted Nodes|%d", alloc.Metrics.NodesExhausted),
300			fmt.Sprintf("Allocation Time|%s", alloc.Metrics.AllocationTime),
301			fmt.Sprintf("Failures|%d", alloc.Metrics.CoalescedFailures))
302	}
303
304	return formatKV(basic), nil
305}
306
307func formatAllocNetworkInfo(alloc *api.Allocation) string {
308	nw := alloc.AllocatedResources.Shared.Networks[0]
309	addrs := make([]string, len(nw.DynamicPorts)+len(nw.ReservedPorts)+1)
310	addrs[0] = "Label|Dynamic|Address"
311	portFmt := func(port *api.Port, dyn string) string {
312		s := fmt.Sprintf("%s|%s|%s:%d", port.Label, dyn, nw.IP, port.Value)
313		if port.To > 0 {
314			s += fmt.Sprintf(" -> %d", port.To)
315		}
316		return s
317	}
318	for idx, port := range nw.DynamicPorts {
319		addrs[idx+1] = portFmt(&port, "yes")
320	}
321	for idx, port := range nw.ReservedPorts {
322		addrs[idx+1+len(nw.DynamicPorts)] = portFmt(&port, "yes")
323	}
324
325	var mode string
326	if nw.Mode != "" {
327		mode = fmt.Sprintf(" (mode = %q)", nw.Mode)
328	}
329
330	return fmt.Sprintf("Allocation Addresses%s\n%s", mode, formatList(addrs))
331}
332
333// futureEvalTimePretty returns when the eval is eligible to reschedule
334// relative to current time, based on the WaitUntil field
335func futureEvalTimePretty(evalID string, client *api.Client) string {
336	evaluation, _, err := client.Evaluations().Info(evalID, nil)
337	// Eval time is not a critical output,
338	// don't return it on errors, if its not set or already in the past
339	if err != nil || evaluation.WaitUntil.IsZero() || time.Now().After(evaluation.WaitUntil) {
340		return ""
341	}
342	return prettyTimeDiff(evaluation.WaitUntil, time.Now())
343}
344
345// outputTaskDetails prints task details for each task in the allocation,
346// optionally printing verbose statistics if displayStats is set
347func (c *AllocStatusCommand) outputTaskDetails(alloc *api.Allocation, stats *api.AllocResourceUsage, displayStats bool) {
348	for task := range c.sortedTaskStateIterator(alloc.TaskStates) {
349		state := alloc.TaskStates[task]
350		c.Ui.Output(c.Colorize().Color(fmt.Sprintf("\n[bold]Task %q is %q[reset]", task, state.State)))
351		c.outputTaskResources(alloc, task, stats, displayStats)
352		c.Ui.Output("")
353		c.outputTaskStatus(state)
354	}
355}
356
357func formatTaskTimes(t time.Time) string {
358	if t.IsZero() {
359		return "N/A"
360	}
361
362	return formatTime(t)
363}
364
365// outputTaskStatus prints out a list of the most recent events for the given
366// task state.
367func (c *AllocStatusCommand) outputTaskStatus(state *api.TaskState) {
368	basic := []string{
369		fmt.Sprintf("Started At|%s", formatTaskTimes(state.StartedAt)),
370		fmt.Sprintf("Finished At|%s", formatTaskTimes(state.FinishedAt)),
371		fmt.Sprintf("Total Restarts|%d", state.Restarts),
372		fmt.Sprintf("Last Restart|%s", formatTaskTimes(state.LastRestart))}
373
374	c.Ui.Output("Task Events:")
375	c.Ui.Output(formatKV(basic))
376	c.Ui.Output("")
377
378	c.Ui.Output("Recent Events:")
379	events := make([]string, len(state.Events)+1)
380	events[0] = "Time|Type|Description"
381
382	size := len(state.Events)
383	for i, event := range state.Events {
384		msg := event.DisplayMessage
385		if msg == "" {
386			msg = buildDisplayMessage(event)
387		}
388		formattedTime := formatUnixNanoTime(event.Time)
389		events[size-i] = fmt.Sprintf("%s|%s|%s", formattedTime, event.Type, msg)
390		// Reverse order so we are sorted by time
391	}
392	c.Ui.Output(formatList(events))
393}
394
395func buildDisplayMessage(event *api.TaskEvent) string {
396	// Build up the description based on the event type.
397	var desc string
398	switch event.Type {
399	case api.TaskSetup:
400		desc = event.Message
401	case api.TaskStarted:
402		desc = "Task started by client"
403	case api.TaskReceived:
404		desc = "Task received by client"
405	case api.TaskFailedValidation:
406		if event.ValidationError != "" {
407			desc = event.ValidationError
408		} else {
409			desc = "Validation of task failed"
410		}
411	case api.TaskSetupFailure:
412		if event.SetupError != "" {
413			desc = event.SetupError
414		} else {
415			desc = "Task setup failed"
416		}
417	case api.TaskDriverFailure:
418		if event.DriverError != "" {
419			desc = event.DriverError
420		} else {
421			desc = "Failed to start task"
422		}
423	case api.TaskDownloadingArtifacts:
424		desc = "Client is downloading artifacts"
425	case api.TaskArtifactDownloadFailed:
426		if event.DownloadError != "" {
427			desc = event.DownloadError
428		} else {
429			desc = "Failed to download artifacts"
430		}
431	case api.TaskKilling:
432		if event.KillReason != "" {
433			desc = fmt.Sprintf("Killing task: %v", event.KillReason)
434		} else if event.KillTimeout != 0 {
435			desc = fmt.Sprintf("Sent interrupt. Waiting %v before force killing", event.KillTimeout)
436		} else {
437			desc = "Sent interrupt"
438		}
439	case api.TaskKilled:
440		if event.KillError != "" {
441			desc = event.KillError
442		} else {
443			desc = "Task successfully killed"
444		}
445	case api.TaskTerminated:
446		var parts []string
447		parts = append(parts, fmt.Sprintf("Exit Code: %d", event.ExitCode))
448
449		if event.Signal != 0 {
450			parts = append(parts, fmt.Sprintf("Signal: %d", event.Signal))
451		}
452
453		if event.Message != "" {
454			parts = append(parts, fmt.Sprintf("Exit Message: %q", event.Message))
455		}
456		desc = strings.Join(parts, ", ")
457	case api.TaskRestarting:
458		in := fmt.Sprintf("Task restarting in %v", time.Duration(event.StartDelay))
459		if event.RestartReason != "" && event.RestartReason != restarts.ReasonWithinPolicy {
460			desc = fmt.Sprintf("%s - %s", event.RestartReason, in)
461		} else {
462			desc = in
463		}
464	case api.TaskNotRestarting:
465		if event.RestartReason != "" {
466			desc = event.RestartReason
467		} else {
468			desc = "Task exceeded restart policy"
469		}
470	case api.TaskSiblingFailed:
471		if event.FailedSibling != "" {
472			desc = fmt.Sprintf("Task's sibling %q failed", event.FailedSibling)
473		} else {
474			desc = "Task's sibling failed"
475		}
476	case api.TaskSignaling:
477		sig := event.TaskSignal
478		reason := event.TaskSignalReason
479
480		if sig == "" && reason == "" {
481			desc = "Task being sent a signal"
482		} else if sig == "" {
483			desc = reason
484		} else if reason == "" {
485			desc = fmt.Sprintf("Task being sent signal %v", sig)
486		} else {
487			desc = fmt.Sprintf("Task being sent signal %v: %v", sig, reason)
488		}
489	case api.TaskRestartSignal:
490		if event.RestartReason != "" {
491			desc = event.RestartReason
492		} else {
493			desc = "Task signaled to restart"
494		}
495	case api.TaskDriverMessage:
496		desc = event.DriverMessage
497	case api.TaskLeaderDead:
498		desc = "Leader Task in Group dead"
499	default:
500		desc = event.Message
501	}
502
503	return desc
504}
505
506// outputTaskResources prints the task resources for the passed task and if
507// displayStats is set, verbose resource usage statistics
508func (c *AllocStatusCommand) outputTaskResources(alloc *api.Allocation, task string, stats *api.AllocResourceUsage, displayStats bool) {
509	resource, ok := alloc.TaskResources[task]
510	if !ok {
511		return
512	}
513
514	c.Ui.Output("Task Resources")
515	var addr []string
516	for _, nw := range resource.Networks {
517		ports := append(nw.DynamicPorts, nw.ReservedPorts...)
518		for _, port := range ports {
519			addr = append(addr, fmt.Sprintf("%v: %v:%v\n", port.Label, nw.IP, port.Value))
520		}
521	}
522
523	var resourcesOutput []string
524	resourcesOutput = append(resourcesOutput, "CPU|Memory|Disk|Addresses")
525	firstAddr := ""
526	if len(addr) > 0 {
527		firstAddr = addr[0]
528	}
529
530	// Display the rolled up stats. If possible prefer the live statistics
531	cpuUsage := strconv.Itoa(*resource.CPU)
532	memUsage := humanize.IBytes(uint64(*resource.MemoryMB * bytesPerMegabyte))
533	var deviceStats []*api.DeviceGroupStats
534
535	if stats != nil {
536		if ru, ok := stats.Tasks[task]; ok && ru != nil && ru.ResourceUsage != nil {
537			if cs := ru.ResourceUsage.CpuStats; cs != nil {
538				cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cs.TotalTicks), cpuUsage)
539			}
540			if ms := ru.ResourceUsage.MemoryStats; ms != nil {
541				memUsage = fmt.Sprintf("%v/%v", humanize.IBytes(ms.RSS), memUsage)
542			}
543			deviceStats = ru.ResourceUsage.DeviceStats
544		}
545	}
546	resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v MHz|%v|%v|%v",
547		cpuUsage,
548		memUsage,
549		humanize.IBytes(uint64(*alloc.Resources.DiskMB*bytesPerMegabyte)),
550		firstAddr))
551	for i := 1; i < len(addr); i++ {
552		resourcesOutput = append(resourcesOutput, fmt.Sprintf("||||%v", addr[i]))
553	}
554	c.Ui.Output(formatListWithSpaces(resourcesOutput))
555
556	if len(deviceStats) > 0 {
557		c.Ui.Output("")
558		c.Ui.Output("Device Stats")
559		c.Ui.Output(formatList(getDeviceResources(deviceStats)))
560	}
561
562	if stats != nil {
563		if ru, ok := stats.Tasks[task]; ok && ru != nil && displayStats && ru.ResourceUsage != nil {
564			c.Ui.Output("")
565			c.outputVerboseResourceUsage(task, ru.ResourceUsage)
566		}
567	}
568}
569
570// outputVerboseResourceUsage outputs the verbose resource usage for the passed
571// task
572func (c *AllocStatusCommand) outputVerboseResourceUsage(task string, resourceUsage *api.ResourceUsage) {
573	memoryStats := resourceUsage.MemoryStats
574	cpuStats := resourceUsage.CpuStats
575	deviceStats := resourceUsage.DeviceStats
576
577	if memoryStats != nil && len(memoryStats.Measured) > 0 {
578		c.Ui.Output("Memory Stats")
579
580		// Sort the measured stats
581		sort.Strings(memoryStats.Measured)
582
583		var measuredStats []string
584		for _, measured := range memoryStats.Measured {
585			switch measured {
586			case "RSS":
587				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.RSS))
588			case "Cache":
589				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Cache))
590			case "Swap":
591				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Swap))
592			case "Usage":
593				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Usage))
594			case "Max Usage":
595				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.MaxUsage))
596			case "Kernel Usage":
597				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelUsage))
598			case "Kernel Max Usage":
599				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelMaxUsage))
600			}
601		}
602
603		out := make([]string, 2)
604		out[0] = strings.Join(memoryStats.Measured, "|")
605		out[1] = strings.Join(measuredStats, "|")
606		c.Ui.Output(formatList(out))
607		c.Ui.Output("")
608	}
609
610	if cpuStats != nil && len(cpuStats.Measured) > 0 {
611		c.Ui.Output("CPU Stats")
612
613		// Sort the measured stats
614		sort.Strings(cpuStats.Measured)
615
616		var measuredStats []string
617		for _, measured := range cpuStats.Measured {
618			switch measured {
619			case "Percent":
620				percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64)
621				measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent))
622			case "Throttled Periods":
623				measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledPeriods))
624			case "Throttled Time":
625				measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledTime))
626			case "User Mode":
627				percent := strconv.FormatFloat(cpuStats.UserMode, 'f', 2, 64)
628				measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent))
629			case "System Mode":
630				percent := strconv.FormatFloat(cpuStats.SystemMode, 'f', 2, 64)
631				measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent))
632			}
633		}
634
635		out := make([]string, 2)
636		out[0] = strings.Join(cpuStats.Measured, "|")
637		out[1] = strings.Join(measuredStats, "|")
638		c.Ui.Output(formatList(out))
639	}
640
641	if len(deviceStats) > 0 {
642		c.Ui.Output("")
643		c.Ui.Output("Device Stats")
644
645		printDeviceStats(c.Ui, deviceStats)
646	}
647}
648
649// shortTaskStatus prints out the current state of each task.
650func (c *AllocStatusCommand) shortTaskStatus(alloc *api.Allocation) {
651	tasks := make([]string, 0, len(alloc.TaskStates)+1)
652	tasks = append(tasks, "Name|State|Last Event|Time")
653	for task := range c.sortedTaskStateIterator(alloc.TaskStates) {
654		state := alloc.TaskStates[task]
655		lastState := state.State
656		var lastEvent, lastTime string
657
658		l := len(state.Events)
659		if l != 0 {
660			last := state.Events[l-1]
661			lastEvent = last.Type
662			lastTime = formatUnixNanoTime(last.Time)
663		}
664
665		tasks = append(tasks, fmt.Sprintf("%s|%s|%s|%s",
666			task, lastState, lastEvent, lastTime))
667	}
668
669	c.Ui.Output(c.Colorize().Color("\n[bold]Tasks[reset]"))
670	c.Ui.Output(formatList(tasks))
671}
672
673// sortedTaskStateIterator is a helper that takes the task state map and returns a
674// channel that returns the keys in a sorted order.
675func (c *AllocStatusCommand) sortedTaskStateIterator(m map[string]*api.TaskState) <-chan string {
676	output := make(chan string, len(m))
677	keys := make([]string, len(m))
678	i := 0
679	for k := range m {
680		keys[i] = k
681		i++
682	}
683	sort.Strings(keys)
684
685	for _, key := range keys {
686		output <- key
687	}
688
689	close(output)
690	return output
691}
692