1package connect 2 3import ( 4 "fmt" 5 "strings" 6 "time" 7 8 consulapi "github.com/hashicorp/consul/api" 9 "github.com/hashicorp/nomad/api" 10 "github.com/hashicorp/nomad/e2e/framework" 11 "github.com/hashicorp/nomad/helper/uuid" 12 "github.com/hashicorp/nomad/jobspec" 13 "github.com/hashicorp/nomad/testutil" 14 "github.com/kr/pretty" 15 "github.com/stretchr/testify/require" 16) 17 18// TestMultiServiceConnect tests running multiple envoy sidecars in the same allocation. 19func (tc *ConnectE2ETest) TestMultiServiceConnect(f *framework.F) { 20 t := f.T() 21 uuid := uuid.Generate() 22 jobID := "connect" + uuid[0:8] 23 tc.jobIds = append(tc.jobIds, jobID) 24 jobapi := tc.Nomad().Jobs() 25 26 job, err := jobspec.ParseFile("connect/input/multi-service.nomad") 27 require.NoError(t, err) 28 job.ID = &jobID 29 30 resp, _, err := jobapi.Register(job, nil) 31 require.NoError(t, err) 32 require.NotNil(t, resp) 33 require.Zero(t, resp.Warnings) 34 35EVAL: 36 qopts := &api.QueryOptions{ 37 WaitIndex: resp.EvalCreateIndex, 38 } 39 evalapi := tc.Nomad().Evaluations() 40 eval, qmeta, err := evalapi.Info(resp.EvalID, qopts) 41 require.NoError(t, err) 42 qopts.WaitIndex = qmeta.LastIndex 43 44 switch eval.Status { 45 case "pending": 46 goto EVAL 47 case "complete": 48 // Ok! 49 case "failed", "canceled", "blocked": 50 require.Failf(t, "expected complete status", "eval %s\n%s", eval.Status, pretty.Sprint(eval)) 51 default: 52 require.Failf(t, "expected complete status", "unknown eval status: %s\n%s", eval.Status, pretty.Sprint(eval)) 53 } 54 55 // Assert there were 0 placement failures 56 require.Zero(t, eval.FailedTGAllocs, pretty.Sprint(eval.FailedTGAllocs)) 57 require.Len(t, eval.QueuedAllocations, 1, pretty.Sprint(eval.QueuedAllocations)) 58 59 // Assert allocs are running 60 for i := 0; i < 20; i++ { 61 allocs, qmeta, err := evalapi.Allocations(eval.ID, qopts) 62 require.NoError(t, err) 63 require.Len(t, allocs, 1) 64 qopts.WaitIndex = qmeta.LastIndex 65 66 running := 0 67 for _, alloc := range allocs { 68 switch alloc.ClientStatus { 69 case "running": 70 running++ 71 case "pending": 72 // keep trying 73 default: 74 require.Failf(t, "alloc failed", "alloc: %s", pretty.Sprint(alloc)) 75 } 76 } 77 78 if running == len(allocs) { 79 break 80 } 81 82 time.Sleep(500 * time.Millisecond) 83 } 84 85 allocs, _, err := evalapi.Allocations(eval.ID, qopts) 86 require.NoError(t, err) 87 allocIDs := make(map[string]bool, 1) 88 for _, a := range allocs { 89 if a.ClientStatus != "running" || a.DesiredStatus != "run" { 90 require.Failf(t, "expected running status", "alloc %s (%s) terminal; client=%s desired=%s", a.TaskGroup, a.ID, a.ClientStatus, a.DesiredStatus) 91 } 92 allocIDs[a.ID] = true 93 } 94 95 // Check Consul service health 96 agentapi := tc.Consul().Agent() 97 98 failing := map[string]*consulapi.AgentCheck{} 99 testutil.WaitForResultRetries(60, func() (bool, error) { 100 defer time.Sleep(time.Second) 101 102 checks, err := agentapi.Checks() 103 require.NoError(t, err) 104 105 // Filter out checks for other services 106 for cid, check := range checks { 107 found := false 108 for allocID := range allocIDs { 109 if strings.Contains(check.ServiceID, allocID) { 110 found = true 111 break 112 } 113 } 114 115 if !found { 116 delete(checks, cid) 117 } 118 } 119 120 // Ensure checks are all passing 121 failing = map[string]*consulapi.AgentCheck{} 122 for _, check := range checks { 123 if check.Status != "passing" { 124 failing[check.CheckID] = check 125 break 126 } 127 } 128 129 if len(failing) == 0 { 130 return true, nil 131 } 132 133 t.Logf("still %d checks not passing", len(failing)) 134 return false, fmt.Errorf("checks are not passing %v %v", len(failing), pretty.Sprint(failing)) 135 }, func(e error) { 136 require.NoError(t, err) 137 }) 138 139 require.Len(t, failing, 0, pretty.Sprint(failing)) 140} 141