1package scheduler
2
3import (
4	"fmt"
5	"reflect"
6	"runtime"
7	"testing"
8
9	"github.com/hashicorp/nomad/nomad/mock"
10	"github.com/hashicorp/nomad/nomad/structs"
11	"github.com/stretchr/testify/require"
12)
13
14func BenchmarkServiceStack_With_ComputedClass(b *testing.B) {
15	// Key doesn't escape computed node class.
16	benchmarkServiceStack_MetaKeyConstraint(b, "key", 5000, 64)
17}
18
19func BenchmarkServiceStack_WithOut_ComputedClass(b *testing.B) {
20	// Key escapes computed node class.
21	benchmarkServiceStack_MetaKeyConstraint(b, "unique.key", 5000, 64)
22}
23
24// benchmarkServiceStack_MetaKeyConstraint creates the passed number of nodes
25// and sets the meta data key to have nodePartitions number of values. It then
26// benchmarks the stack by selecting a job that constrains against one of the
27// partitions.
28func benchmarkServiceStack_MetaKeyConstraint(b *testing.B, key string, numNodes, nodePartitions int) {
29	_, ctx := testContext(b)
30	stack := NewGenericStack(false, ctx)
31
32	// Create 4 classes of nodes.
33	nodes := make([]*structs.Node, numNodes)
34	for i := 0; i < numNodes; i++ {
35		n := mock.Node()
36		n.Meta[key] = fmt.Sprintf("%d", i%nodePartitions)
37		nodes[i] = n
38	}
39	stack.SetNodes(nodes)
40
41	// Create a job whose constraint meets two node classes.
42	job := mock.Job()
43	job.Constraints[0] = &structs.Constraint{
44		LTarget: fmt.Sprintf("${meta.%v}", key),
45		RTarget: "1",
46		Operand: "<",
47	}
48	stack.SetJob(job)
49
50	b.ResetTimer()
51	selectOptions := &SelectOptions{}
52	for i := 0; i < b.N; i++ {
53		stack.Select(job.TaskGroups[0], selectOptions)
54	}
55}
56
57func TestServiceStack_SetNodes(t *testing.T) {
58	_, ctx := testContext(t)
59	stack := NewGenericStack(false, ctx)
60
61	nodes := []*structs.Node{
62		mock.Node(),
63		mock.Node(),
64		mock.Node(),
65		mock.Node(),
66		mock.Node(),
67		mock.Node(),
68		mock.Node(),
69		mock.Node(),
70	}
71	stack.SetNodes(nodes)
72
73	// Check that our scan limit is updated
74	if stack.limit.limit != 3 {
75		t.Fatalf("bad limit %d", stack.limit.limit)
76	}
77
78	out := collectFeasible(stack.source)
79	if !reflect.DeepEqual(out, nodes) {
80		t.Fatalf("bad: %#v", out)
81	}
82}
83
84func TestServiceStack_SetJob(t *testing.T) {
85	_, ctx := testContext(t)
86	stack := NewGenericStack(false, ctx)
87
88	job := mock.Job()
89	stack.SetJob(job)
90
91	if stack.binPack.priority != job.Priority {
92		t.Fatalf("bad")
93	}
94	if !reflect.DeepEqual(stack.jobConstraint.constraints, job.Constraints) {
95		t.Fatalf("bad")
96	}
97}
98
99func TestServiceStack_Select_Size(t *testing.T) {
100	_, ctx := testContext(t)
101	nodes := []*structs.Node{
102		mock.Node(),
103	}
104	stack := NewGenericStack(false, ctx)
105	stack.SetNodes(nodes)
106
107	job := mock.Job()
108	stack.SetJob(job)
109	selectOptions := &SelectOptions{}
110	node := stack.Select(job.TaskGroups[0], selectOptions)
111	if node == nil {
112		t.Fatalf("missing node %#v", ctx.Metrics())
113	}
114
115	// Note: On Windows time.Now currently has a best case granularity of 1ms.
116	// We skip the following assertion on Windows because this test usually
117	// runs too fast to measure an allocation time on Windows.
118	met := ctx.Metrics()
119	if runtime.GOOS != "windows" && met.AllocationTime == 0 {
120		t.Fatalf("missing time")
121	}
122}
123
124func TestServiceStack_Select_PreferringNodes(t *testing.T) {
125	_, ctx := testContext(t)
126	nodes := []*structs.Node{
127		mock.Node(),
128	}
129	stack := NewGenericStack(false, ctx)
130	stack.SetNodes(nodes)
131
132	job := mock.Job()
133	stack.SetJob(job)
134
135	// Create a preferred node
136	preferredNode := mock.Node()
137	prefNodes := []*structs.Node{preferredNode}
138	selectOptions := &SelectOptions{PreferredNodes: prefNodes}
139	option := stack.Select(job.TaskGroups[0], selectOptions)
140	if option == nil {
141		t.Fatalf("missing node %#v", ctx.Metrics())
142	}
143	if option.Node.ID != preferredNode.ID {
144		t.Fatalf("expected: %v, actual: %v", option.Node.ID, preferredNode.ID)
145	}
146
147	// Make sure select doesn't have a side effect on preferred nodes
148	require.Equal(t, prefNodes, selectOptions.PreferredNodes)
149
150	// Change the preferred node's kernel to windows and ensure the allocations
151	// are placed elsewhere
152	preferredNode1 := preferredNode.Copy()
153	preferredNode1.Attributes["kernel.name"] = "windows"
154	preferredNode1.ComputeClass()
155	prefNodes1 := []*structs.Node{preferredNode1}
156	selectOptions = &SelectOptions{PreferredNodes: prefNodes1}
157	option = stack.Select(job.TaskGroups[0], selectOptions)
158	if option == nil {
159		t.Fatalf("missing node %#v", ctx.Metrics())
160	}
161
162	if option.Node.ID != nodes[0].ID {
163		t.Fatalf("expected: %#v, actual: %#v", nodes[0], option.Node)
164	}
165	require.Equal(t, prefNodes1, selectOptions.PreferredNodes)
166}
167
168func TestServiceStack_Select_MetricsReset(t *testing.T) {
169	_, ctx := testContext(t)
170	nodes := []*structs.Node{
171		mock.Node(),
172		mock.Node(),
173		mock.Node(),
174		mock.Node(),
175	}
176	stack := NewGenericStack(false, ctx)
177	stack.SetNodes(nodes)
178
179	job := mock.Job()
180	stack.SetJob(job)
181	selectOptions := &SelectOptions{}
182	n1 := stack.Select(job.TaskGroups[0], selectOptions)
183	m1 := ctx.Metrics()
184	if n1 == nil {
185		t.Fatalf("missing node %#v", m1)
186	}
187
188	if m1.NodesEvaluated != 2 {
189		t.Fatalf("should only be 2")
190	}
191
192	n2 := stack.Select(job.TaskGroups[0], selectOptions)
193	m2 := ctx.Metrics()
194	if n2 == nil {
195		t.Fatalf("missing node %#v", m2)
196	}
197
198	// If we don't reset, this would be 4
199	if m2.NodesEvaluated != 2 {
200		t.Fatalf("should only be 2")
201	}
202}
203
204func TestServiceStack_Select_DriverFilter(t *testing.T) {
205	_, ctx := testContext(t)
206	nodes := []*structs.Node{
207		mock.Node(),
208		mock.Node(),
209	}
210	zero := nodes[0]
211	zero.Attributes["driver.foo"] = "1"
212	if err := zero.ComputeClass(); err != nil {
213		t.Fatalf("ComputedClass() failed: %v", err)
214	}
215
216	stack := NewGenericStack(false, ctx)
217	stack.SetNodes(nodes)
218
219	job := mock.Job()
220	job.TaskGroups[0].Tasks[0].Driver = "foo"
221	stack.SetJob(job)
222
223	selectOptions := &SelectOptions{}
224	node := stack.Select(job.TaskGroups[0], selectOptions)
225	if node == nil {
226		t.Fatalf("missing node %#v", ctx.Metrics())
227	}
228
229	if node.Node != zero {
230		t.Fatalf("bad")
231	}
232}
233
234func TestServiceStack_Select_ConstraintFilter(t *testing.T) {
235	_, ctx := testContext(t)
236	nodes := []*structs.Node{
237		mock.Node(),
238		mock.Node(),
239	}
240	zero := nodes[0]
241	zero.Attributes["kernel.name"] = "freebsd"
242	if err := zero.ComputeClass(); err != nil {
243		t.Fatalf("ComputedClass() failed: %v", err)
244	}
245
246	stack := NewGenericStack(false, ctx)
247	stack.SetNodes(nodes)
248
249	job := mock.Job()
250	job.Constraints[0].RTarget = "freebsd"
251	stack.SetJob(job)
252	selectOptions := &SelectOptions{}
253	node := stack.Select(job.TaskGroups[0], selectOptions)
254	if node == nil {
255		t.Fatalf("missing node %#v", ctx.Metrics())
256	}
257
258	if node.Node != zero {
259		t.Fatalf("bad")
260	}
261
262	met := ctx.Metrics()
263	if met.NodesFiltered != 1 {
264		t.Fatalf("bad: %#v", met)
265	}
266	if met.ClassFiltered["linux-medium-pci"] != 1 {
267		t.Fatalf("bad: %#v", met)
268	}
269	if met.ConstraintFiltered["${attr.kernel.name} = freebsd"] != 1 {
270		t.Fatalf("bad: %#v", met)
271	}
272}
273
274func TestServiceStack_Select_BinPack_Overflow(t *testing.T) {
275	_, ctx := testContext(t)
276	nodes := []*structs.Node{
277		mock.Node(),
278		mock.Node(),
279	}
280	zero := nodes[0]
281	one := nodes[1]
282	one.ReservedResources = &structs.NodeReservedResources{
283		Cpu: structs.NodeReservedCpuResources{
284			CpuShares: one.NodeResources.Cpu.CpuShares,
285		},
286	}
287
288	stack := NewGenericStack(false, ctx)
289	stack.SetNodes(nodes)
290
291	job := mock.Job()
292	stack.SetJob(job)
293	selectOptions := &SelectOptions{}
294	node := stack.Select(job.TaskGroups[0], selectOptions)
295	ctx.Metrics().PopulateScoreMetaData()
296	if node == nil {
297		t.Fatalf("missing node %#v", ctx.Metrics())
298	}
299
300	if node.Node != zero {
301		t.Fatalf("bad")
302	}
303
304	met := ctx.Metrics()
305	if met.NodesExhausted != 1 {
306		t.Fatalf("bad: %#v", met)
307	}
308	if met.ClassExhausted["linux-medium-pci"] != 1 {
309		t.Fatalf("bad: %#v", met)
310	}
311	// Expect score metadata for one node
312	if len(met.ScoreMetaData) != 1 {
313		t.Fatalf("bad: %#v", met)
314	}
315}
316
317func TestSystemStack_SetNodes(t *testing.T) {
318	_, ctx := testContext(t)
319	stack := NewSystemStack(ctx)
320
321	nodes := []*structs.Node{
322		mock.Node(),
323		mock.Node(),
324		mock.Node(),
325		mock.Node(),
326		mock.Node(),
327		mock.Node(),
328		mock.Node(),
329		mock.Node(),
330	}
331	stack.SetNodes(nodes)
332
333	out := collectFeasible(stack.source)
334	if !reflect.DeepEqual(out, nodes) {
335		t.Fatalf("bad: %#v", out)
336	}
337}
338
339func TestSystemStack_SetJob(t *testing.T) {
340	_, ctx := testContext(t)
341	stack := NewSystemStack(ctx)
342
343	job := mock.Job()
344	stack.SetJob(job)
345
346	if stack.binPack.priority != job.Priority {
347		t.Fatalf("bad")
348	}
349	if !reflect.DeepEqual(stack.jobConstraint.constraints, job.Constraints) {
350		t.Fatalf("bad")
351	}
352}
353
354func TestSystemStack_Select_Size(t *testing.T) {
355	_, ctx := testContext(t)
356	nodes := []*structs.Node{mock.Node()}
357	stack := NewSystemStack(ctx)
358	stack.SetNodes(nodes)
359
360	job := mock.Job()
361	stack.SetJob(job)
362	selectOptions := &SelectOptions{}
363	node := stack.Select(job.TaskGroups[0], selectOptions)
364	if node == nil {
365		t.Fatalf("missing node %#v", ctx.Metrics())
366	}
367
368	// Note: On Windows time.Now currently has a best case granularity of 1ms.
369	// We skip the following assertion on Windows because this test usually
370	// runs too fast to measure an allocation time on Windows.
371	met := ctx.Metrics()
372	if runtime.GOOS != "windows" && met.AllocationTime == 0 {
373		t.Fatalf("missing time")
374	}
375}
376
377func TestSystemStack_Select_MetricsReset(t *testing.T) {
378	_, ctx := testContext(t)
379	nodes := []*structs.Node{
380		mock.Node(),
381		mock.Node(),
382		mock.Node(),
383		mock.Node(),
384	}
385	stack := NewSystemStack(ctx)
386	stack.SetNodes(nodes)
387
388	job := mock.Job()
389	stack.SetJob(job)
390	selectOptions := &SelectOptions{}
391	n1 := stack.Select(job.TaskGroups[0], selectOptions)
392	m1 := ctx.Metrics()
393	if n1 == nil {
394		t.Fatalf("missing node %#v", m1)
395	}
396
397	if m1.NodesEvaluated != 1 {
398		t.Fatalf("should only be 1")
399	}
400
401	n2 := stack.Select(job.TaskGroups[0], selectOptions)
402	m2 := ctx.Metrics()
403	if n2 == nil {
404		t.Fatalf("missing node %#v", m2)
405	}
406
407	// If we don't reset, this would be 2
408	if m2.NodesEvaluated != 1 {
409		t.Fatalf("should only be 2")
410	}
411}
412
413func TestSystemStack_Select_DriverFilter(t *testing.T) {
414	_, ctx := testContext(t)
415	nodes := []*structs.Node{
416		mock.Node(),
417	}
418	zero := nodes[0]
419	zero.Attributes["driver.foo"] = "1"
420
421	stack := NewSystemStack(ctx)
422	stack.SetNodes(nodes)
423
424	job := mock.Job()
425	job.TaskGroups[0].Tasks[0].Driver = "foo"
426	stack.SetJob(job)
427
428	selectOptions := &SelectOptions{}
429	node := stack.Select(job.TaskGroups[0], selectOptions)
430	if node == nil {
431		t.Fatalf("missing node %#v", ctx.Metrics())
432	}
433
434	if node.Node != zero {
435		t.Fatalf("bad")
436	}
437
438	zero.Attributes["driver.foo"] = "0"
439	if err := zero.ComputeClass(); err != nil {
440		t.Fatalf("ComputedClass() failed: %v", err)
441	}
442
443	stack = NewSystemStack(ctx)
444	stack.SetNodes(nodes)
445	stack.SetJob(job)
446	node = stack.Select(job.TaskGroups[0], selectOptions)
447	if node != nil {
448		t.Fatalf("node not filtered %#v", node)
449	}
450}
451
452func TestSystemStack_Select_ConstraintFilter(t *testing.T) {
453	_, ctx := testContext(t)
454	nodes := []*structs.Node{
455		mock.Node(),
456		mock.Node(),
457	}
458	zero := nodes[1]
459	zero.Attributes["kernel.name"] = "freebsd"
460	if err := zero.ComputeClass(); err != nil {
461		t.Fatalf("ComputedClass() failed: %v", err)
462	}
463
464	stack := NewSystemStack(ctx)
465	stack.SetNodes(nodes)
466
467	job := mock.Job()
468	job.Constraints[0].RTarget = "freebsd"
469	stack.SetJob(job)
470
471	selectOptions := &SelectOptions{}
472	node := stack.Select(job.TaskGroups[0], selectOptions)
473	if node == nil {
474		t.Fatalf("missing node %#v", ctx.Metrics())
475	}
476
477	if node.Node != zero {
478		t.Fatalf("bad")
479	}
480
481	met := ctx.Metrics()
482	if met.NodesFiltered != 1 {
483		t.Fatalf("bad: %#v", met)
484	}
485	if met.ClassFiltered["linux-medium-pci"] != 1 {
486		t.Fatalf("bad: %#v", met)
487	}
488	if met.ConstraintFiltered["${attr.kernel.name} = freebsd"] != 1 {
489		t.Fatalf("bad: %#v", met)
490	}
491}
492
493func TestSystemStack_Select_BinPack_Overflow(t *testing.T) {
494	_, ctx := testContext(t)
495	nodes := []*structs.Node{
496		mock.Node(),
497		mock.Node(),
498	}
499	zero := nodes[0]
500	zero.ReservedResources = &structs.NodeReservedResources{
501		Cpu: structs.NodeReservedCpuResources{
502			CpuShares: zero.NodeResources.Cpu.CpuShares,
503		},
504	}
505	one := nodes[1]
506
507	stack := NewSystemStack(ctx)
508	stack.SetNodes(nodes)
509
510	job := mock.Job()
511	stack.SetJob(job)
512
513	selectOptions := &SelectOptions{}
514	node := stack.Select(job.TaskGroups[0], selectOptions)
515	ctx.Metrics().PopulateScoreMetaData()
516	if node == nil {
517		t.Fatalf("missing node %#v", ctx.Metrics())
518	}
519
520	if node.Node != one {
521		t.Fatalf("bad")
522	}
523
524	met := ctx.Metrics()
525	if met.NodesExhausted != 1 {
526		t.Fatalf("bad: %#v", met)
527	}
528	if met.ClassExhausted["linux-medium-pci"] != 1 {
529		t.Fatalf("bad: %#v", met)
530	}
531	// Should have two scores, one from bin packing and one from normalization
532	if len(met.ScoreMetaData) != 1 {
533		t.Fatalf("bad: %#v", met)
534	}
535}
536