1package command
2
3import (
4	"fmt"
5	"os"
6	"os/exec"
7	"strings"
8
9	capi "github.com/hashicorp/consul/api"
10	hclog "github.com/hashicorp/go-hclog"
11	vapi "github.com/hashicorp/vault/api"
12	"github.com/mitchellh/cli"
13)
14
15func RunCommandFactory(meta Meta) cli.CommandFactory {
16	return func() (cli.Command, error) {
17		return &Run{Meta: meta}, nil
18	}
19}
20
21type Run struct {
22	Meta
23}
24
25func (c *Run) Help() string {
26	helpText := `
27Usage: nomad-e2e run (<provider>/<name>)...
28
29  Two modes exist when using the run command.
30
31  When no arguments are given to the run command, it will launch
32  the e2e test suite against the Nomad cluster specified by the
33  NOMAD_ADDR environment variable. If this is not set, it defaults
34  to 'http://localhost:4646'
35
36  Multiple arguments may be given to specify one or more environments to
37  provision and run the e2e tests against. These are given in the form of
38  <provider>/<name>. Globs are support, for example 'aws/*' would run tests
39  against all of the environments under the aws provider. When using this mode,
40  all of the provision flags are supported.
41
42General Options:
43
44` + generalOptionsUsage() + `
45
46Run Options:
47
48  -run regex
49    Sets a regular expression for what tests to run. Uses '/' as a separator
50	to allow hierarchy between Suite/Case/Test.
51
52	Example '-run MyTestSuite' would only run tests under the MyTestSuite suite.
53
54  -slow
55    If set, will only run test suites marked as slow.
56`
57	return strings.TrimSpace(helpText)
58}
59
60func (c *Run) Synopsis() string {
61	return "Runs the e2e test suite"
62}
63
64func (c *Run) Run(args []string) int {
65	var envPath string
66	var nomadBinary string
67	var tfPath string
68	var slow bool
69	var run string
70	cmdFlags := c.FlagSet("run")
71	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
72	cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs")
73	cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "")
74	cmdFlags.StringVar(&tfPath, "tf-path", "", "")
75	cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases")
76	cmdFlags.BoolVar(&slow, "slow", false, "Toggle slow running suites")
77
78	if err := cmdFlags.Parse(args); err != nil {
79		c.logger.Error("failed to parse flags", "error", err)
80		return 1
81	}
82	if c.verbose {
83		c.logger.SetLevel(hclog.Debug)
84	}
85
86	args = cmdFlags.Args()
87
88	if len(args) == 0 {
89		c.logger.Info("no environments specified, running test suite locally")
90		report, err := c.runTest(&runOpts{
91			run:     run,
92			slow:    slow,
93			verbose: c.verbose,
94		})
95		if err != nil {
96			c.logger.Error("failed to run test suite", "error", err)
97			return 1
98		}
99		if report.TotalFailedTests > 0 {
100			c.Ui.Error("***FAILED***")
101			c.Ui.Error(report.Summary())
102			return 1
103		}
104		c.Ui.Output("PASSED!")
105		if c.verbose {
106			c.Ui.Output(report.Summary())
107		}
108		return 0
109	}
110
111	environments := []*environment{}
112	for _, e := range args {
113		if len(strings.Split(e, "/")) != 2 {
114			c.logger.Error("argument should be formated as <provider>/<environment>", "args", e)
115			return 1
116		}
117		envs, err := envsFromGlob(envPath, e, tfPath, c.logger)
118		if err != nil {
119			c.logger.Error("failed to build environment", "environment", e, "error", err)
120			return 1
121		}
122		environments = append(environments, envs...)
123
124	}
125
126	// Use go-getter to fetch the nomad binary
127	nomadPath, err := fetchBinary(nomadBinary)
128	defer os.RemoveAll(nomadPath)
129	if err != nil {
130		c.logger.Error("failed to fetch nomad binary", "error", err)
131		return 1
132	}
133
134	envCount := len(environments)
135	c.logger.Debug("starting tests", "totalEnvironments", envCount)
136	failedEnvs := map[string]*TestReport{}
137	for i, env := range environments {
138		logger := c.logger.With("name", env.name, "provider", env.provider)
139		logger.Debug("provisioning environment")
140		results, err := env.provision(nomadPath)
141		if err != nil {
142			logger.Error("failed to provision environment", "error", err)
143			return 1
144		}
145
146		opts := &runOpts{
147			provider:   env.provider,
148			env:        env.name,
149			slow:       slow,
150			run:        run,
151			verbose:    c.verbose,
152			nomadAddr:  results.nomadAddr,
153			consulAddr: results.consulAddr,
154			vaultAddr:  results.vaultAddr,
155		}
156
157		var report *TestReport
158		if report, err = c.runTest(opts); err != nil {
159			logger.Error("failed to run tests against environment", "error", err)
160			return 1
161		}
162		if report.TotalFailedTests > 0 {
163			c.Ui.Error(fmt.Sprintf("[%d/%d] %s: ***FAILED***", i+1, envCount, env.canonicalName()))
164			c.Ui.Error(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary()))
165			failedEnvs[env.canonicalName()] = report
166		}
167
168		c.Ui.Output(fmt.Sprintf("[%d/%d] %s: PASSED!", i+1, envCount, env.canonicalName()))
169		if c.verbose {
170			c.Ui.Output(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary()))
171		}
172	}
173
174	if len(failedEnvs) > 0 {
175		c.Ui.Error(fmt.Sprintf("The following environments ***FAILED***"))
176		for name, report := range failedEnvs {
177			c.Ui.Error(fmt.Sprintf("  [%s]: %d out of %d suite failures",
178				name, report.TotalFailedSuites, report.TotalSuites))
179		}
180		return 1
181	}
182	c.Ui.Output("All Environments PASSED!")
183	return 0
184}
185
186func (c *Run) runTest(opts *runOpts) (*TestReport, error) {
187	goBin, err := exec.LookPath("go")
188	if err != nil {
189		return nil, err
190	}
191
192	cmd := exec.Command(goBin, opts.goArgs()...)
193	cmd.Env = opts.goEnv()
194	out, err := cmd.StdoutPipe()
195	if err != nil {
196		return nil, err
197	}
198
199	err = cmd.Start()
200	if err != nil {
201		return nil, err
202	}
203
204	err = cmd.Wait()
205	if err != nil {
206		// should command fail, log here then proceed to generate test report
207		// to report more informative info about which tests fail
208		c.logger.Error("test command failed", "error", err)
209	}
210
211	dec := NewDecoder(out)
212	report, err := dec.Decode(c.logger.Named("run.gotest"))
213	if err != nil {
214		return nil, err
215	}
216
217	return report, nil
218
219}
220
221// runOpts contains fields used to build the arguments and environment variabled
222// nessicary to run go test and initialize the e2e framework
223type runOpts struct {
224	nomadAddr  string
225	consulAddr string
226	vaultAddr  string
227	provider   string
228	env        string
229	run        string
230	local      bool
231	slow       bool
232	verbose    bool
233}
234
235// goArgs returns the list of arguments passed to the go command to start the
236// e2e test framework
237func (opts *runOpts) goArgs() []string {
238	a := []string{
239		"test",
240		"-json",
241	}
242
243	if opts.run != "" {
244		a = append(a, "-run=TestE2E/"+opts.run)
245	}
246
247	a = append(a, []string{
248		"github.com/hashicorp/nomad/e2e",
249		"-env=" + opts.env,
250		"-env.provider=" + opts.provider,
251	}...)
252
253	if opts.slow {
254		a = append(a, "-slow")
255	}
256
257	if opts.local {
258		a = append(a, "-local")
259	}
260	return a
261}
262
263// goEnv returns the list of environment variabled passed to the go command to start
264// the e2e test framework
265func (opts *runOpts) goEnv() []string {
266	env := append(os.Environ(), "NOMAD_E2E=1")
267	if opts.nomadAddr != "" {
268		env = append(env, "NOMAD_ADDR="+opts.nomadAddr)
269	}
270	if opts.consulAddr != "" {
271		env = append(env, fmt.Sprintf("%s=%s", capi.HTTPAddrEnvName, opts.consulAddr))
272	}
273	if opts.vaultAddr != "" {
274		env = append(env, fmt.Sprintf("%s=%s", vapi.EnvVaultAddress, opts.consulAddr))
275	}
276
277	return env
278}
279