1// Copyright 2018 The Prometheus Authors
2// Licensed under the Apache License, Version 2.0 (the "License");
3// you may not use this file except in compliance with the License.
4// You may obtain a copy of the License at
5//
6// http://www.apache.org/licenses/LICENSE-2.0
7//
8// Unless required by applicable law or agreed to in writing, software
9// distributed under the License is distributed on an "AS IS" BASIS,
10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11// See the License for the specific language governing permissions and
12// limitations under the License.
13
14package tsdb
15
16import (
17	"encoding/json"
18	"io"
19	"io/ioutil"
20	"os"
21	"path/filepath"
22
23	"github.com/go-kit/log"
24	"github.com/go-kit/log/level"
25	"github.com/pkg/errors"
26
27	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
28	"github.com/prometheus/prometheus/tsdb/fileutil"
29)
30
31// repairBadIndexVersion repairs an issue in index and meta.json persistence introduced in
32// commit 129773b41a565fde5156301e37f9a87158030443.
33func repairBadIndexVersion(logger log.Logger, dir string) error {
34	// All blocks written by Prometheus 2.1 with a meta.json version of 2 are affected.
35	// We must actually set the index file version to 2 and revert the meta.json version back to 1.
36	dirs, err := blockDirs(dir)
37	if err != nil {
38		return errors.Wrapf(err, "list block dirs in %q", dir)
39	}
40
41	tmpFiles := make([]string, 0, len(dirs))
42	defer func() {
43		for _, tmp := range tmpFiles {
44			if err := os.RemoveAll(tmp); err != nil {
45				level.Error(logger).Log("msg", "remove tmp file", "err", err.Error())
46			}
47		}
48	}()
49
50	for _, d := range dirs {
51		meta, err := readBogusMetaFile(d)
52		if err != nil {
53			level.Error(logger).Log("msg", "failed to read meta.json for a block during repair process; skipping", "dir", d, "err", err)
54			continue
55		}
56		if meta.Version == metaVersion1 {
57			level.Info(logger).Log(
58				"msg", "Found healthy block",
59				"mint", meta.MinTime,
60				"maxt", meta.MaxTime,
61				"ulid", meta.ULID,
62			)
63			continue
64		}
65		level.Info(logger).Log(
66			"msg", "Fixing broken block",
67			"mint", meta.MinTime,
68			"maxt", meta.MaxTime,
69			"ulid", meta.ULID,
70		)
71
72		repl, err := os.Create(filepath.Join(d, "index.repaired"))
73		if err != nil {
74			return errors.Wrapf(err, "create index.repaired for block dir: %v", d)
75		}
76		tmpFiles = append(tmpFiles, repl.Name())
77
78		broken, err := os.Open(filepath.Join(d, indexFilename))
79		if err != nil {
80			return errors.Wrapf(err, "open broken index for block dir: %v", d)
81		}
82		if _, err := io.Copy(repl, broken); err != nil {
83			return errors.Wrapf(err, "copy content of index to index.repaired for block dir: %v", d)
84		}
85
86		// Set the 5th byte to 2 to indicate the correct file format version.
87		if _, err := repl.WriteAt([]byte{2}, 4); err != nil {
88			return tsdb_errors.NewMulti(
89				errors.Wrapf(err, "rewrite of index.repaired for block dir: %v", d),
90				errors.Wrap(repl.Close(), "close"),
91			).Err()
92		}
93		if err := repl.Sync(); err != nil {
94			return tsdb_errors.NewMulti(
95				errors.Wrapf(err, "sync of index.repaired for block dir: %v", d),
96				errors.Wrap(repl.Close(), "close"),
97			).Err()
98		}
99		if err := repl.Close(); err != nil {
100			return errors.Wrapf(repl.Close(), "close repaired index for block dir: %v", d)
101		}
102		if err := broken.Close(); err != nil {
103			return errors.Wrapf(repl.Close(), "close broken index for block dir: %v", d)
104		}
105		if err := fileutil.Replace(repl.Name(), broken.Name()); err != nil {
106			return errors.Wrapf(repl.Close(), "replaced broken index with index.repaired for block dir: %v", d)
107		}
108		// Reset version of meta.json to 1.
109		meta.Version = metaVersion1
110		if _, err := writeMetaFile(logger, d, meta); err != nil {
111			return errors.Wrapf(repl.Close(), "write meta for block dir: %v", d)
112		}
113	}
114	return nil
115}
116
117func readBogusMetaFile(dir string) (*BlockMeta, error) {
118	b, err := ioutil.ReadFile(filepath.Join(dir, metaFilename))
119	if err != nil {
120		return nil, err
121	}
122	var m BlockMeta
123
124	if err := json.Unmarshal(b, &m); err != nil {
125		return nil, err
126	}
127	if m.Version != metaVersion1 && m.Version != 2 {
128		return nil, errors.Errorf("unexpected meta file version %d", m.Version)
129	}
130	return &m, nil
131}
132