1// Copyright (c) 2015-2021 MinIO, Inc.
2//
3// This file is part of MinIO Object Storage stack
4//
5// This program is free software: you can redistribute it and/or modify
6// it under the terms of the GNU Affero General Public License as published by
7// the Free Software Foundation, either version 3 of the License, or
8// (at your option) any later version.
9//
10// This program is distributed in the hope that it will be useful
11// but WITHOUT ANY WARRANTY; without even the implied warranty of
12// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13// GNU Affero General Public License for more details.
14//
15// You should have received a copy of the GNU Affero General Public License
16// along with this program.  If not, see <http://www.gnu.org/licenses/>.
17
18package cmd
19
20import (
21	"errors"
22	"fmt"
23	"math"
24	"strings"
25	"time"
26
27	humanize "github.com/dustin/go-humanize"
28	"github.com/fatih/color"
29	json "github.com/minio/colorjson"
30	"github.com/minio/madmin-go"
31	"github.com/minio/mc/pkg/probe"
32	"github.com/minio/pkg/console"
33)
34
35const (
36	lineWidth = 80
37)
38
39var (
40	hColOrder = []col{colRed, colYellow, colGreen}
41	hColTable = map[int][]int{
42		1: {0, -1, 1},
43		2: {0, 1, 2},
44		3: {1, 2, 3},
45		4: {1, 2, 4},
46		5: {1, 3, 5},
47		6: {2, 4, 6},
48		7: {2, 4, 7},
49		8: {2, 5, 8},
50	}
51)
52
53func getHColCode(surplusShards, parityShards int) (c col, err error) {
54	if parityShards < 1 || parityShards > 8 || surplusShards > parityShards {
55		return c, fmt.Errorf("Invalid parity shard count/surplus shard count given")
56	}
57	if surplusShards < 0 {
58		return colGrey, err
59	}
60	colRow := hColTable[parityShards]
61	for index, val := range colRow {
62		if val != -1 && surplusShards <= val {
63			return hColOrder[index], err
64		}
65	}
66	return c, fmt.Errorf("cannot get a heal color code")
67}
68
69type uiData struct {
70	Bucket, Prefix string
71	Client         *madmin.AdminClient
72	ClientToken    string
73	ForceStart     bool
74	HealOpts       *madmin.HealOpts
75	LastItem       *hri
76
77	// Total time since heal start
78	HealDuration time.Duration
79
80	// Accumulated statistics of heal result records
81	BytesScanned int64
82
83	// Counter for objects, and another counter for all kinds of
84	// items
85	ObjectsScanned, ItemsScanned int64
86
87	// Counters for healed objects and all kinds of healed items
88	ObjectsHealed, ItemsHealed int64
89
90	// Map from online drives to number of objects with that many
91	// online drives.
92	ObjectsByOnlineDrives map[int]int64
93	// Map of health color code to number of objects with that
94	// health color code.
95	HealthCols map[col]int64
96
97	// channel to receive a prompt string to indicate activity on
98	// the terminal
99	CurChan (<-chan string)
100}
101
102func (ui *uiData) updateStats(i madmin.HealResultItem) error {
103	if i.Type == madmin.HealItemObject {
104		// Objects whose size could not be found have -1 size
105		// returned.
106		if i.ObjectSize >= 0 {
107			ui.BytesScanned += i.ObjectSize
108		}
109
110		ui.ObjectsScanned++
111	}
112	ui.ItemsScanned++
113
114	beforeUp, afterUp := i.GetOnlineCounts()
115	if afterUp > beforeUp {
116		if i.Type == madmin.HealItemObject {
117			ui.ObjectsHealed++
118		}
119		ui.ItemsHealed++
120	}
121	ui.ObjectsByOnlineDrives[afterUp]++
122
123	// Update health color stats:
124
125	// Fetch health color after heal:
126	var err error
127	var afterCol col
128	h := newHRI(&i)
129	switch h.Type {
130	case madmin.HealItemMetadata, madmin.HealItemBucket:
131		_, afterCol, err = h.getReplicatedFileHCCChange()
132	default:
133		_, afterCol, err = h.getObjectHCCChange()
134	}
135	if err != nil {
136		return err
137	}
138
139	ui.HealthCols[afterCol]++
140	return nil
141}
142
143func (ui *uiData) updateDuration(s *madmin.HealTaskStatus) {
144	ui.HealDuration = UTCNow().Sub(s.StartTime)
145}
146
147func (ui *uiData) getProgress() (oCount, objSize, duration string) {
148	oCount = humanize.Comma(ui.ObjectsScanned)
149
150	duration = ui.HealDuration.Round(time.Second).String()
151
152	bytesScanned := float64(ui.BytesScanned)
153
154	// Compute unit for object size
155	magnitudes := []float64{1 << 10, 1 << 20, 1 << 30, 1 << 40, 1 << 50, 1 << 60}
156	units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"}
157	var i int
158	for i = 0; i < len(magnitudes); i++ {
159		if bytesScanned <= magnitudes[i] {
160			break
161		}
162	}
163	numUnits := int(bytesScanned * (1 << 10) / magnitudes[i])
164	objSize = fmt.Sprintf("%d %s", numUnits, units[i])
165	return
166}
167
168func (ui *uiData) getPercentsNBars() (p map[col]float64, b map[col]string) {
169	// barChar, emptyBarChar := "█", "░"
170	barChar, emptyBarChar := "█", " "
171	barLen := 12
172	sum := float64(ui.ItemsScanned)
173	cols := []col{colGrey, colRed, colYellow, colGreen}
174
175	p = make(map[col]float64, len(cols))
176	b = make(map[col]string, len(cols))
177	var filledLen int
178	for _, col := range cols {
179		v := float64(ui.HealthCols[col])
180		if sum == 0 {
181			p[col] = 0
182			filledLen = 0
183		} else {
184			p[col] = v * 100 / sum
185			// round up the filled part
186			filledLen = int(math.Ceil(float64(barLen) * v / sum))
187		}
188		b[col] = strings.Repeat(barChar, filledLen) +
189			strings.Repeat(emptyBarChar, barLen-filledLen)
190	}
191	return
192}
193
194func (ui *uiData) printItemsQuietly(s *madmin.HealTaskStatus) (err error) {
195	lpad := func(s col) string {
196		return fmt.Sprintf("%-6s", string(s))
197	}
198	rpad := func(s col) string {
199		return fmt.Sprintf("%6s", string(s))
200	}
201	printColStr := func(before, after col) {
202		console.PrintC("[" + lpad(before) + " -> " + rpad(after) + "] ")
203	}
204
205	var b, a col
206	for _, item := range s.Items {
207		h := newHRI(&item)
208		switch h.Type {
209		case madmin.HealItemMetadata, madmin.HealItemBucket:
210			b, a, err = h.getReplicatedFileHCCChange()
211		default:
212			b, a, err = h.getObjectHCCChange()
213		}
214		if err != nil {
215			return err
216		}
217		printColStr(b, a)
218		hrStr := h.getHealResultStr()
219		switch h.Type {
220		case madmin.HealItemMetadata, madmin.HealItemBucketMetadata:
221			console.PrintC(fmt.Sprintln("**", hrStr, "**"))
222		default:
223			console.PrintC(hrStr, "\n")
224		}
225	}
226	return nil
227}
228
229func (ui *uiData) printStatsQuietly(s *madmin.HealTaskStatus) {
230	totalObjects, totalSize, totalTime := ui.getProgress()
231
232	healedStr := fmt.Sprintf("Healed:\t%s/%s objects; %s in %s\n",
233		humanize.Comma(ui.ObjectsHealed), totalObjects,
234		totalSize, totalTime)
235
236	console.PrintC(healedStr)
237}
238
239func (ui *uiData) printItemsJSON(s *madmin.HealTaskStatus) (err error) {
240	type healRec struct {
241		Status string `json:"status"`
242		Error  string `json:"error,omitempty"`
243		Type   string `json:"type"`
244		Name   string `json:"name"`
245		Before struct {
246			Color     string                 `json:"color"`
247			Offline   int                    `json:"offline"`
248			Online    int                    `json:"online"`
249			Missing   int                    `json:"missing"`
250			Corrupted int                    `json:"corrupted"`
251			Drives    []madmin.HealDriveInfo `json:"drives"`
252		} `json:"before"`
253		After struct {
254			Color     string                 `json:"color"`
255			Offline   int                    `json:"offline"`
256			Online    int                    `json:"online"`
257			Missing   int                    `json:"missing"`
258			Corrupted int                    `json:"corrupted"`
259			Drives    []madmin.HealDriveInfo `json:"drives"`
260		} `json:"after"`
261		Size int64 `json:"size"`
262	}
263	makeHR := func(h *hri) (r healRec) {
264		r.Status = "success"
265		r.Type, r.Name = h.getHRTypeAndName()
266
267		var b, a col
268		var err error
269		switch h.Type {
270		case madmin.HealItemMetadata, madmin.HealItemBucket:
271			b, a, err = h.getReplicatedFileHCCChange()
272		default:
273			if h.Type == madmin.HealItemObject {
274				r.Size = h.ObjectSize
275			}
276			b, a, err = h.getObjectHCCChange()
277		}
278		if err != nil {
279			r.Error = err.Error()
280		}
281		r.Before.Color = strings.ToLower(string(b))
282		r.After.Color = strings.ToLower(string(a))
283		r.Before.Online, r.After.Online = h.GetOnlineCounts()
284		r.Before.Missing, r.After.Missing = h.GetMissingCounts()
285		r.Before.Corrupted, r.After.Corrupted = h.GetCorruptedCounts()
286		r.Before.Offline, r.After.Offline = h.GetOfflineCounts()
287		r.Before.Drives = h.Before.Drives
288		r.After.Drives = h.After.Drives
289		return r
290	}
291
292	for _, item := range s.Items {
293		h := newHRI(&item)
294		jsonBytes, err := json.MarshalIndent(makeHR(h), "", " ")
295		fatalIf(probe.NewError(err), "Unable to marshal to JSON.")
296		console.Println(string(jsonBytes))
297	}
298	return nil
299}
300
301func (ui *uiData) printStatsJSON(s *madmin.HealTaskStatus) {
302	var summary struct {
303		Status         string `json:"status"`
304		Error          string `json:"error,omitempty"`
305		Type           string `json:"type"`
306		ObjectsScanned int64  `json:"objects_scanned"`
307		ObjectsHealed  int64  `json:"objects_healed"`
308		ItemsScanned   int64  `json:"items_scanned"`
309		ItemsHealed    int64  `json:"items_healed"`
310		Size           int64  `json:"size"`
311		ElapsedTime    int64  `json:"duration"`
312	}
313
314	summary.Status = "success"
315	summary.Type = "summary"
316
317	summary.ObjectsScanned = ui.ObjectsScanned
318	summary.ObjectsHealed = ui.ObjectsHealed
319	summary.ItemsScanned = ui.ItemsScanned
320	summary.ItemsHealed = ui.ItemsHealed
321	summary.Size = ui.BytesScanned
322	summary.ElapsedTime = int64(ui.HealDuration.Round(time.Second).Seconds())
323
324	jBytes, err := json.MarshalIndent(summary, "", " ")
325	fatalIf(probe.NewError(err), "Unable to marshal to JSON.")
326	console.Println(string(jBytes))
327}
328
329func (ui *uiData) updateUI(s *madmin.HealTaskStatus) (err error) {
330	itemCount := len(s.Items)
331	h := ui.LastItem
332	if itemCount > 0 {
333		item := s.Items[itemCount-1]
334		h = newHRI(&item)
335		ui.LastItem = h
336	}
337	scannedStr := "** waiting for status from server **"
338	if h != nil {
339		scannedStr = lineTrunc(h.makeHealEntityString(), lineWidth-len("Scanned: "))
340	}
341
342	totalObjects, totalSize, totalTime := ui.getProgress()
343	healedStr := fmt.Sprintf("%s/%s objects; %s in %s",
344		humanize.Comma(ui.ObjectsHealed), totalObjects,
345		totalSize, totalTime)
346
347	console.Print(console.Colorize("HealUpdateUI", fmt.Sprintf(" %s", <-ui.CurChan)))
348	console.PrintC(fmt.Sprintf("  %s\n", scannedStr))
349	console.PrintC(fmt.Sprintf("    %s\n", healedStr))
350
351	dspOrder := []col{colGreen, colYellow, colRed, colGrey}
352	printColors := []*color.Color{}
353	for _, c := range dspOrder {
354		printColors = append(printColors, getPrintCol(c))
355	}
356	t := console.NewTable(printColors, []bool{false, true, true}, 4)
357
358	percentMap, barMap := ui.getPercentsNBars()
359	cellText := make([][]string, len(dspOrder))
360	for i := range cellText {
361		cellText[i] = []string{
362			string(dspOrder[i]),
363			fmt.Sprint(humanize.Comma(ui.HealthCols[dspOrder[i]])),
364			fmt.Sprintf("%5.1f%% %s", percentMap[dspOrder[i]], barMap[dspOrder[i]]),
365		}
366	}
367
368	t.DisplayTable(cellText)
369	return nil
370}
371
372func (ui *uiData) UpdateDisplay(s *madmin.HealTaskStatus) (err error) {
373	// Update state
374	ui.updateDuration(s)
375	for _, i := range s.Items {
376		ui.updateStats(i)
377	}
378
379	// Update display
380	switch {
381	case globalJSON:
382		err = ui.printItemsJSON(s)
383	case globalQuiet:
384		err = ui.printItemsQuietly(s)
385	default:
386		err = ui.updateUI(s)
387	}
388	return
389}
390
391func (ui *uiData) healResumeMsg(aliasedURL string) string {
392	var flags string
393	if ui.HealOpts.Recursive {
394		flags += "--recursive "
395	}
396	if ui.HealOpts.DryRun {
397		flags += "--dry-run "
398	}
399	return fmt.Sprintf("Healing is backgrounded, to resume watching use `mc admin heal %s %s`", flags, aliasedURL)
400}
401
402func (ui *uiData) DisplayAndFollowHealStatus(aliasedURL string) (res madmin.HealTaskStatus, err error) {
403	quitMsg := ui.healResumeMsg(aliasedURL)
404
405	firstIter := true
406	for {
407		select {
408		case <-globalContext.Done():
409			return res, errors.New(quitMsg)
410		default:
411			_, res, err = ui.Client.Heal(globalContext, ui.Bucket, ui.Prefix, *ui.HealOpts,
412				ui.ClientToken, ui.ForceStart, false)
413			if err != nil {
414				return res, err
415			}
416			if firstIter {
417				firstIter = false
418			} else {
419				if !globalQuiet && !globalJSON {
420					console.RewindLines(8)
421				}
422			}
423			err = ui.UpdateDisplay(&res)
424			if err != nil {
425				return res, err
426			}
427
428			if res.Summary == "finished" {
429				if globalJSON {
430					ui.printStatsJSON(&res)
431				} else if globalQuiet {
432					ui.printStatsQuietly(&res)
433				}
434				return res, nil
435			}
436
437			if res.Summary == "stopped" {
438				return res, fmt.Errorf("Heal had an error - %s", res.FailureDetail)
439			}
440
441			time.Sleep(time.Second)
442		}
443	}
444}
445