1package connect
2
3import (
4	"fmt"
5	"strings"
6	"time"
7
8	consulapi "github.com/hashicorp/consul/api"
9	"github.com/hashicorp/nomad/api"
10	"github.com/hashicorp/nomad/e2e/framework"
11	"github.com/hashicorp/nomad/helper/uuid"
12	"github.com/hashicorp/nomad/jobspec"
13	"github.com/hashicorp/nomad/testutil"
14	"github.com/kr/pretty"
15	"github.com/stretchr/testify/require"
16)
17
18// TestMultiServiceConnect tests running multiple envoy sidecars in the same allocation.
19func (tc *ConnectE2ETest) TestMultiServiceConnect(f *framework.F) {
20	t := f.T()
21	uuid := uuid.Generate()
22	jobID := "connect" + uuid[0:8]
23	tc.jobIds = append(tc.jobIds, jobID)
24	jobapi := tc.Nomad().Jobs()
25
26	job, err := jobspec.ParseFile("connect/input/multi-service.nomad")
27	require.NoError(t, err)
28	job.ID = &jobID
29
30	resp, _, err := jobapi.Register(job, nil)
31	require.NoError(t, err)
32	require.NotNil(t, resp)
33	require.Zero(t, resp.Warnings)
34
35EVAL:
36	qopts := &api.QueryOptions{
37		WaitIndex: resp.EvalCreateIndex,
38	}
39	evalapi := tc.Nomad().Evaluations()
40	eval, qmeta, err := evalapi.Info(resp.EvalID, qopts)
41	require.NoError(t, err)
42	qopts.WaitIndex = qmeta.LastIndex
43
44	switch eval.Status {
45	case "pending":
46		goto EVAL
47	case "complete":
48		// Ok!
49	case "failed", "canceled", "blocked":
50		require.Failf(t, "expected complete status", "eval %s\n%s", eval.Status, pretty.Sprint(eval))
51	default:
52		require.Failf(t, "expected complete status", "unknown eval status: %s\n%s", eval.Status, pretty.Sprint(eval))
53	}
54
55	// Assert there were 0 placement failures
56	require.Zero(t, eval.FailedTGAllocs, pretty.Sprint(eval.FailedTGAllocs))
57	require.Len(t, eval.QueuedAllocations, 1, pretty.Sprint(eval.QueuedAllocations))
58
59	// Assert allocs are running
60	for i := 0; i < 20; i++ {
61		allocs, qmeta, err := evalapi.Allocations(eval.ID, qopts)
62		require.NoError(t, err)
63		require.Len(t, allocs, 1)
64		qopts.WaitIndex = qmeta.LastIndex
65
66		running := 0
67		for _, alloc := range allocs {
68			switch alloc.ClientStatus {
69			case "running":
70				running++
71			case "pending":
72				// keep trying
73			default:
74				require.Failf(t, "alloc failed", "alloc: %s", pretty.Sprint(alloc))
75			}
76		}
77
78		if running == len(allocs) {
79			break
80		}
81
82		time.Sleep(500 * time.Millisecond)
83	}
84
85	allocs, _, err := evalapi.Allocations(eval.ID, qopts)
86	require.NoError(t, err)
87	allocIDs := make(map[string]bool, 1)
88	for _, a := range allocs {
89		if a.ClientStatus != "running" || a.DesiredStatus != "run" {
90			require.Failf(t, "expected running status", "alloc %s (%s) terminal; client=%s desired=%s", a.TaskGroup, a.ID, a.ClientStatus, a.DesiredStatus)
91		}
92		allocIDs[a.ID] = true
93	}
94
95	// Check Consul service health
96	agentapi := tc.Consul().Agent()
97
98	failing := map[string]*consulapi.AgentCheck{}
99	testutil.WaitForResultRetries(60, func() (bool, error) {
100		defer time.Sleep(time.Second)
101
102		checks, err := agentapi.Checks()
103		require.NoError(t, err)
104
105		// Filter out checks for other services
106		for cid, check := range checks {
107			found := false
108			for allocID := range allocIDs {
109				if strings.Contains(check.ServiceID, allocID) {
110					found = true
111					break
112				}
113			}
114
115			if !found {
116				delete(checks, cid)
117			}
118		}
119
120		// Ensure checks are all passing
121		failing = map[string]*consulapi.AgentCheck{}
122		for _, check := range checks {
123			if check.Status != "passing" {
124				failing[check.CheckID] = check
125				break
126			}
127		}
128
129		if len(failing) == 0 {
130			return true, nil
131		}
132
133		t.Logf("still %d checks not passing", len(failing))
134		return false, fmt.Errorf("checks are not passing %v %v", len(failing), pretty.Sprint(failing))
135	}, func(e error) {
136		require.NoError(t, err)
137	})
138
139	require.Len(t, failing, 0, pretty.Sprint(failing))
140}
141