1/*
2Copyright 2018 The Kubernetes Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8    http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17package state
18
19import (
20	"io/ioutil"
21	"os"
22	"reflect"
23	"strings"
24	"testing"
25
26	"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
27	"k8s.io/kubernetes/pkg/kubelet/cm/containermap"
28	testutil "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/testing"
29	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
30)
31
32const testingCheckpoint = "cpumanager_checkpoint_test"
33
34func TestCheckpointStateRestore(t *testing.T) {
35	testCases := []struct {
36		description       string
37		checkpointContent string
38		policyName        string
39		initialContainers containermap.ContainerMap
40		expectedError     string
41		expectedState     *stateMemory
42	}{
43		{
44			"Restore non-existing checkpoint",
45			"",
46			"none",
47			containermap.ContainerMap{},
48			"",
49			&stateMemory{},
50		},
51		{
52			"Restore default cpu set",
53			`{
54				"policyName": "none",
55				"defaultCPUSet": "4-6",
56				"entries": {},
57				"checksum": 354655845
58			}`,
59			"none",
60			containermap.ContainerMap{},
61			"",
62			&stateMemory{
63				defaultCPUSet: cpuset.NewCPUSet(4, 5, 6),
64			},
65		},
66		{
67			"Restore valid checkpoint",
68			`{
69				"policyName": "none",
70				"defaultCPUSet": "1-3",
71				"entries": {
72					"pod": {
73						"container1": "4-6",
74						"container2": "1-3"
75					}
76				},
77				"checksum": 3610638499
78			}`,
79			"none",
80			containermap.ContainerMap{},
81			"",
82			&stateMemory{
83				assignments: ContainerCPUAssignments{
84					"pod": map[string]cpuset.CPUSet{
85						"container1": cpuset.NewCPUSet(4, 5, 6),
86						"container2": cpuset.NewCPUSet(1, 2, 3),
87					},
88				},
89				defaultCPUSet: cpuset.NewCPUSet(1, 2, 3),
90			},
91		},
92		{
93			"Restore checkpoint with invalid checksum",
94			`{
95				"policyName": "none",
96				"defaultCPUSet": "4-6",
97				"entries": {},
98				"checksum": 1337
99			}`,
100			"none",
101			containermap.ContainerMap{},
102			"checkpoint is corrupted",
103			&stateMemory{},
104		},
105		{
106			"Restore checkpoint with invalid JSON",
107			`{`,
108			"none",
109			containermap.ContainerMap{},
110			"unexpected end of JSON input",
111			&stateMemory{},
112		},
113		{
114			"Restore checkpoint with invalid policy name",
115			`{
116				"policyName": "other",
117				"defaultCPUSet": "1-3",
118				"entries": {},
119				"checksum": 1394507217
120			}`,
121			"none",
122			containermap.ContainerMap{},
123			`configured policy "none" differs from state checkpoint policy "other"`,
124			&stateMemory{},
125		},
126		{
127			"Restore checkpoint with unparsable default cpu set",
128			`{
129				"policyName": "none",
130				"defaultCPUSet": "1.3",
131				"entries": {},
132				"checksum": 3021697696
133			}`,
134			"none",
135			containermap.ContainerMap{},
136			`could not parse default cpu set "1.3": strconv.Atoi: parsing "1.3": invalid syntax`,
137			&stateMemory{},
138		},
139		{
140			"Restore checkpoint with unparsable assignment entry",
141			`{
142				"policyName": "none",
143				"defaultCPUSet": "1-3",
144				"entries": {
145					"pod": {
146						"container1": "4-6",
147						"container2": "asd"
148					}
149				},
150				"checksum": 962272150
151			}`,
152			"none",
153			containermap.ContainerMap{},
154			`could not parse cpuset "asd" for container "container2" in pod "pod": strconv.Atoi: parsing "asd": invalid syntax`,
155			&stateMemory{},
156		},
157		{
158			"Restore checkpoint from checkpoint with v1 checksum",
159			`{
160				"policyName": "none",
161				"defaultCPUSet": "1-3",
162				"checksum": 1694838852
163			}`,
164			"none",
165			containermap.ContainerMap{},
166			"",
167			&stateMemory{
168				defaultCPUSet: cpuset.NewCPUSet(1, 2, 3),
169			},
170		},
171		{
172			"Restore checkpoint with migration",
173			`{
174				"policyName": "none",
175				"defaultCPUSet": "1-3",
176				"entries": {
177					"containerID1": "4-6",
178					"containerID2": "1-3"
179				},
180				"checksum": 3680390589
181			}`,
182			"none",
183			func() containermap.ContainerMap {
184				cm := containermap.NewContainerMap()
185				cm.Add("pod", "container1", "containerID1")
186				cm.Add("pod", "container2", "containerID2")
187				return cm
188			}(),
189			"",
190			&stateMemory{
191				assignments: ContainerCPUAssignments{
192					"pod": map[string]cpuset.CPUSet{
193						"container1": cpuset.NewCPUSet(4, 5, 6),
194						"container2": cpuset.NewCPUSet(1, 2, 3),
195					},
196				},
197				defaultCPUSet: cpuset.NewCPUSet(1, 2, 3),
198			},
199		},
200	}
201
202	// create temp dir
203	testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
204	if err != nil {
205		t.Fatal(err)
206	}
207	defer os.RemoveAll(testingDir)
208	// create checkpoint manager for testing
209	cpm, err := checkpointmanager.NewCheckpointManager(testingDir)
210	if err != nil {
211		t.Fatalf("could not create testing checkpoint manager: %v", err)
212	}
213
214	for _, tc := range testCases {
215		t.Run(tc.description, func(t *testing.T) {
216			// ensure there is no previous checkpoint
217			cpm.RemoveCheckpoint(testingCheckpoint)
218
219			// prepare checkpoint for testing
220			if strings.TrimSpace(tc.checkpointContent) != "" {
221				checkpoint := &testutil.MockCheckpoint{Content: tc.checkpointContent}
222				if err := cpm.CreateCheckpoint(testingCheckpoint, checkpoint); err != nil {
223					t.Fatalf("could not create testing checkpoint: %v", err)
224				}
225			}
226
227			restoredState, err := NewCheckpointState(testingDir, testingCheckpoint, tc.policyName, tc.initialContainers)
228			if err != nil {
229				if strings.TrimSpace(tc.expectedError) != "" {
230					if strings.Contains(err.Error(), "could not restore state from checkpoint") &&
231						strings.Contains(err.Error(), tc.expectedError) {
232						t.Logf("got expected error: %v", err)
233						return
234					}
235				}
236				t.Fatalf("unexpected error while creatng checkpointState: %v", err)
237			}
238
239			// compare state after restoration with the one expected
240			AssertStateEqual(t, restoredState, tc.expectedState)
241		})
242	}
243}
244
245func TestCheckpointStateStore(t *testing.T) {
246	testCases := []struct {
247		description   string
248		expectedState *stateMemory
249	}{
250		{
251			"Store default cpu set",
252			&stateMemory{defaultCPUSet: cpuset.NewCPUSet(1, 2, 3)},
253		},
254		{
255			"Store assignments",
256			&stateMemory{
257				assignments: map[string]map[string]cpuset.CPUSet{
258					"pod": {
259						"container1": cpuset.NewCPUSet(1, 5, 8),
260					},
261				},
262			},
263		},
264	}
265
266	// create temp dir
267	testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
268	if err != nil {
269		t.Fatal(err)
270	}
271	defer os.RemoveAll(testingDir)
272
273	cpm, err := checkpointmanager.NewCheckpointManager(testingDir)
274	if err != nil {
275		t.Fatalf("could not create testing checkpoint manager: %v", err)
276	}
277
278	for _, tc := range testCases {
279		t.Run(tc.description, func(t *testing.T) {
280			// ensure there is no previous checkpoint
281			cpm.RemoveCheckpoint(testingCheckpoint)
282
283			cs1, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
284			if err != nil {
285				t.Fatalf("could not create testing checkpointState instance: %v", err)
286			}
287
288			// set values of cs1 instance so they are stored in checkpoint and can be read by cs2
289			cs1.SetDefaultCPUSet(tc.expectedState.defaultCPUSet)
290			cs1.SetCPUAssignments(tc.expectedState.assignments)
291
292			// restore checkpoint with previously stored values
293			cs2, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
294			if err != nil {
295				t.Fatalf("could not create testing checkpointState instance: %v", err)
296			}
297
298			AssertStateEqual(t, cs2, tc.expectedState)
299		})
300	}
301}
302
303func TestCheckpointStateHelpers(t *testing.T) {
304	testCases := []struct {
305		description   string
306		defaultCPUset cpuset.CPUSet
307		assignments   map[string]map[string]cpuset.CPUSet
308	}{
309		{
310			description:   "One container",
311			defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8),
312			assignments: map[string]map[string]cpuset.CPUSet{
313				"pod": {
314					"c1": cpuset.NewCPUSet(0, 1),
315				},
316			},
317		},
318		{
319			description:   "Two containers",
320			defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8),
321			assignments: map[string]map[string]cpuset.CPUSet{
322				"pod": {
323					"c1": cpuset.NewCPUSet(0, 1),
324					"c2": cpuset.NewCPUSet(2, 3, 4, 5),
325				},
326			},
327		},
328		{
329			description:   "Container without assigned cpus",
330			defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8),
331			assignments: map[string]map[string]cpuset.CPUSet{
332				"pod": {
333					"c1": cpuset.NewCPUSet(),
334				},
335			},
336		},
337	}
338
339	// create temp dir
340	testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
341	if err != nil {
342		t.Fatal(err)
343	}
344	defer os.RemoveAll(testingDir)
345
346	cpm, err := checkpointmanager.NewCheckpointManager(testingDir)
347	if err != nil {
348		t.Fatalf("could not create testing checkpoint manager: %v", err)
349	}
350
351	for _, tc := range testCases {
352		t.Run(tc.description, func(t *testing.T) {
353			// ensure there is no previous checkpoint
354			cpm.RemoveCheckpoint(testingCheckpoint)
355
356			state, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
357			if err != nil {
358				t.Fatalf("could not create testing checkpointState instance: %v", err)
359			}
360			state.SetDefaultCPUSet(tc.defaultCPUset)
361
362			for pod := range tc.assignments {
363				for container, set := range tc.assignments[pod] {
364					state.SetCPUSet(pod, container, set)
365					if cpus, _ := state.GetCPUSet(pod, container); !cpus.Equals(set) {
366						t.Fatalf("state inconsistent, got %q instead of %q", set, cpus)
367					}
368
369					state.Delete(pod, container)
370					if _, ok := state.GetCPUSet(pod, container); ok {
371						t.Fatal("deleted container still existing in state")
372					}
373				}
374			}
375		})
376	}
377}
378
379func TestCheckpointStateClear(t *testing.T) {
380	testCases := []struct {
381		description   string
382		defaultCPUset cpuset.CPUSet
383		assignments   map[string]map[string]cpuset.CPUSet
384	}{
385		{
386			"Valid state",
387			cpuset.NewCPUSet(1, 5, 10),
388			map[string]map[string]cpuset.CPUSet{
389				"pod": {
390					"container1": cpuset.NewCPUSet(1, 4),
391				},
392			},
393		},
394	}
395
396	for _, tc := range testCases {
397		t.Run(tc.description, func(t *testing.T) {
398			// create temp dir
399			testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
400			if err != nil {
401				t.Fatal(err)
402			}
403			defer os.RemoveAll(testingDir)
404
405			state, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
406			if err != nil {
407				t.Fatalf("could not create testing checkpointState instance: %v", err)
408			}
409
410			state.SetDefaultCPUSet(tc.defaultCPUset)
411			state.SetCPUAssignments(tc.assignments)
412
413			state.ClearState()
414			if !cpuset.NewCPUSet().Equals(state.GetDefaultCPUSet()) {
415				t.Fatal("cleared state with non-empty default cpu set")
416			}
417			for pod := range tc.assignments {
418				for container := range tc.assignments[pod] {
419					if _, ok := state.GetCPUSet(pod, container); ok {
420						t.Fatalf("container %q in pod %q with non-default cpu set in cleared state", container, pod)
421					}
422				}
423			}
424		})
425	}
426}
427
428func AssertStateEqual(t *testing.T, sf State, sm State) {
429	cpusetSf := sf.GetDefaultCPUSet()
430	cpusetSm := sm.GetDefaultCPUSet()
431	if !cpusetSf.Equals(cpusetSm) {
432		t.Errorf("State CPUSet mismatch. Have %v, want %v", cpusetSf, cpusetSm)
433	}
434
435	cpuassignmentSf := sf.GetCPUAssignments()
436	cpuassignmentSm := sm.GetCPUAssignments()
437	if !reflect.DeepEqual(cpuassignmentSf, cpuassignmentSm) {
438		t.Errorf("State CPU assignments mismatch. Have %s, want %s", cpuassignmentSf, cpuassignmentSm)
439	}
440}
441