1// Copyright 2017 The go-ethereum Authors
2// This file is part of the go-ethereum library.
3//
4// The go-ethereum library is free software: you can redistribute it and/or modify
5// it under the terms of the GNU Lesser General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// The go-ethereum library is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU Lesser General Public License for more details.
13//
14// You should have received a copy of the GNU Lesser General Public License
15// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16
17package tests
18
19import (
20	"encoding/json"
21	"fmt"
22	"io"
23	"io/ioutil"
24	"os"
25	"path/filepath"
26	"reflect"
27	"regexp"
28	"runtime"
29	"sort"
30	"strings"
31	"testing"
32
33	"github.com/ethereum/go-ethereum/params"
34)
35
36var (
37	baseDir            = filepath.Join(".", "testdata")
38	blockTestDir       = filepath.Join(baseDir, "BlockchainTests")
39	stateTestDir       = filepath.Join(baseDir, "GeneralStateTests")
40	legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests")
41	transactionTestDir = filepath.Join(baseDir, "TransactionTests")
42	rlpTestDir         = filepath.Join(baseDir, "RLPTests")
43	difficultyTestDir  = filepath.Join(baseDir, "BasicTests")
44)
45
46func readJSON(reader io.Reader, value interface{}) error {
47	data, err := ioutil.ReadAll(reader)
48	if err != nil {
49		return fmt.Errorf("error reading JSON file: %v", err)
50	}
51	if err = json.Unmarshal(data, &value); err != nil {
52		if syntaxerr, ok := err.(*json.SyntaxError); ok {
53			line := findLine(data, syntaxerr.Offset)
54			return fmt.Errorf("JSON syntax error at line %v: %v", line, err)
55		}
56		return err
57	}
58	return nil
59}
60
61func readJSONFile(fn string, value interface{}) error {
62	file, err := os.Open(fn)
63	if err != nil {
64		return err
65	}
66	defer file.Close()
67
68	err = readJSON(file, value)
69	if err != nil {
70		return fmt.Errorf("%s in file %s", err.Error(), fn)
71	}
72	return nil
73}
74
75// findLine returns the line number for the given offset into data.
76func findLine(data []byte, offset int64) (line int) {
77	line = 1
78	for i, r := range string(data) {
79		if int64(i) >= offset {
80			return
81		}
82		if r == '\n' {
83			line++
84		}
85	}
86	return
87}
88
89// testMatcher controls skipping and chain config assignment to tests.
90type testMatcher struct {
91	configpat      []testConfig
92	failpat        []testFailure
93	skiploadpat    []*regexp.Regexp
94	slowpat        []*regexp.Regexp
95	runonlylistpat *regexp.Regexp
96}
97
98type testConfig struct {
99	p      *regexp.Regexp
100	config params.ChainConfig
101}
102
103type testFailure struct {
104	p      *regexp.Regexp
105	reason string
106}
107
108// skipShortMode skips tests matching when the -short flag is used.
109func (tm *testMatcher) slow(pattern string) {
110	tm.slowpat = append(tm.slowpat, regexp.MustCompile(pattern))
111}
112
113// skipLoad skips JSON loading of tests matching the pattern.
114func (tm *testMatcher) skipLoad(pattern string) {
115	tm.skiploadpat = append(tm.skiploadpat, regexp.MustCompile(pattern))
116}
117
118// fails adds an expected failure for tests matching the pattern.
119func (tm *testMatcher) fails(pattern string, reason string) {
120	if reason == "" {
121		panic("empty fail reason")
122	}
123	tm.failpat = append(tm.failpat, testFailure{regexp.MustCompile(pattern), reason})
124}
125
126func (tm *testMatcher) runonly(pattern string) {
127	tm.runonlylistpat = regexp.MustCompile(pattern)
128}
129
130// config defines chain config for tests matching the pattern.
131func (tm *testMatcher) config(pattern string, cfg params.ChainConfig) {
132	tm.configpat = append(tm.configpat, testConfig{regexp.MustCompile(pattern), cfg})
133}
134
135// findSkip matches name against test skip patterns.
136func (tm *testMatcher) findSkip(name string) (reason string, skipload bool) {
137	isWin32 := runtime.GOARCH == "386" && runtime.GOOS == "windows"
138	for _, re := range tm.slowpat {
139		if re.MatchString(name) {
140			if testing.Short() {
141				return "skipped in -short mode", false
142			}
143			if isWin32 {
144				return "skipped on 32bit windows", false
145			}
146		}
147	}
148	for _, re := range tm.skiploadpat {
149		if re.MatchString(name) {
150			return "skipped by skipLoad", true
151		}
152	}
153	return "", false
154}
155
156// findConfig returns the chain config matching defined patterns.
157func (tm *testMatcher) findConfig(t *testing.T) *params.ChainConfig {
158	for _, m := range tm.configpat {
159		if m.p.MatchString(t.Name()) {
160			return &m.config
161		}
162	}
163	return new(params.ChainConfig)
164}
165
166// checkFailure checks whether a failure is expected.
167func (tm *testMatcher) checkFailure(t *testing.T, err error) error {
168	failReason := ""
169	for _, m := range tm.failpat {
170		if m.p.MatchString(t.Name()) {
171			failReason = m.reason
172			break
173		}
174	}
175	if failReason != "" {
176		t.Logf("expected failure: %s", failReason)
177		if err != nil {
178			t.Logf("error: %v", err)
179			return nil
180		}
181		return fmt.Errorf("test succeeded unexpectedly")
182	}
183	return err
184}
185
186// walk invokes its runTest argument for all subtests in the given directory.
187//
188// runTest should be a function of type func(t *testing.T, name string, x <TestType>),
189// where TestType is the type of the test contained in test files.
190func (tm *testMatcher) walk(t *testing.T, dir string, runTest interface{}) {
191	// Walk the directory.
192	dirinfo, err := os.Stat(dir)
193	if os.IsNotExist(err) || !dirinfo.IsDir() {
194		fmt.Fprintf(os.Stderr, "can't find test files in %s, did you clone the tests submodule?\n", dir)
195		t.Skip("missing test files")
196	}
197	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
198		name := filepath.ToSlash(strings.TrimPrefix(path, dir+string(filepath.Separator)))
199		if info.IsDir() {
200			if _, skipload := tm.findSkip(name + "/"); skipload {
201				return filepath.SkipDir
202			}
203			return nil
204		}
205		if filepath.Ext(path) == ".json" {
206			t.Run(name, func(t *testing.T) { tm.runTestFile(t, path, name, runTest) })
207		}
208		return nil
209	})
210	if err != nil {
211		t.Fatal(err)
212	}
213}
214
215func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest interface{}) {
216	if r, _ := tm.findSkip(name); r != "" {
217		t.Skip(r)
218	}
219	if tm.runonlylistpat != nil {
220		if !tm.runonlylistpat.MatchString(name) {
221			t.Skip("Skipped by runonly")
222		}
223	}
224	t.Parallel()
225
226	// Load the file as map[string]<testType>.
227	m := makeMapFromTestFunc(runTest)
228	if err := readJSONFile(path, m.Addr().Interface()); err != nil {
229		t.Fatal(err)
230	}
231
232	// Run all tests from the map. Don't wrap in a subtest if there is only one test in the file.
233	keys := sortedMapKeys(m)
234	if len(keys) == 1 {
235		runTestFunc(runTest, t, name, m, keys[0])
236	} else {
237		for _, key := range keys {
238			name := name + "/" + key
239			t.Run(key, func(t *testing.T) {
240				if r, _ := tm.findSkip(name); r != "" {
241					t.Skip(r)
242				}
243				runTestFunc(runTest, t, name, m, key)
244			})
245		}
246	}
247}
248
249func makeMapFromTestFunc(f interface{}) reflect.Value {
250	stringT := reflect.TypeOf("")
251	testingT := reflect.TypeOf((*testing.T)(nil))
252	ftyp := reflect.TypeOf(f)
253	if ftyp.Kind() != reflect.Func || ftyp.NumIn() != 3 || ftyp.NumOut() != 0 || ftyp.In(0) != testingT || ftyp.In(1) != stringT {
254		panic(fmt.Sprintf("bad test function type: want func(*testing.T, string, <TestType>), have %s", ftyp))
255	}
256	testType := ftyp.In(2)
257	mp := reflect.New(reflect.MapOf(stringT, testType))
258	return mp.Elem()
259}
260
261func sortedMapKeys(m reflect.Value) []string {
262	keys := make([]string, m.Len())
263	for i, k := range m.MapKeys() {
264		keys[i] = k.String()
265	}
266	sort.Strings(keys)
267	return keys
268}
269
270func runTestFunc(runTest interface{}, t *testing.T, name string, m reflect.Value, key string) {
271	reflect.ValueOf(runTest).Call([]reflect.Value{
272		reflect.ValueOf(t),
273		reflect.ValueOf(name),
274		m.MapIndex(reflect.ValueOf(key)),
275	})
276}
277
278func TestMatcherRunonlylist(t *testing.T) {
279	t.Parallel()
280	tm := new(testMatcher)
281	tm.runonly("invalid*")
282	tm.walk(t, rlpTestDir, func(t *testing.T, name string, test *RLPTest) {
283		if name[:len("invalidRLPTest.json")] != "invalidRLPTest.json" {
284			t.Fatalf("invalid test found: %s != invalidRLPTest.json", name)
285		}
286	})
287}
288