1/* 2Copyright 2018 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package state 18 19import ( 20 "io/ioutil" 21 "os" 22 "reflect" 23 "strings" 24 "testing" 25 26 "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" 27 "k8s.io/kubernetes/pkg/kubelet/cm/containermap" 28 testutil "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/testing" 29 "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" 30) 31 32const testingCheckpoint = "cpumanager_checkpoint_test" 33 34func TestCheckpointStateRestore(t *testing.T) { 35 testCases := []struct { 36 description string 37 checkpointContent string 38 policyName string 39 initialContainers containermap.ContainerMap 40 expectedError string 41 expectedState *stateMemory 42 }{ 43 { 44 "Restore non-existing checkpoint", 45 "", 46 "none", 47 containermap.ContainerMap{}, 48 "", 49 &stateMemory{}, 50 }, 51 { 52 "Restore default cpu set", 53 `{ 54 "policyName": "none", 55 "defaultCPUSet": "4-6", 56 "entries": {}, 57 "checksum": 354655845 58 }`, 59 "none", 60 containermap.ContainerMap{}, 61 "", 62 &stateMemory{ 63 defaultCPUSet: cpuset.NewCPUSet(4, 5, 6), 64 }, 65 }, 66 { 67 "Restore valid checkpoint", 68 `{ 69 "policyName": "none", 70 "defaultCPUSet": "1-3", 71 "entries": { 72 "pod": { 73 "container1": "4-6", 74 "container2": "1-3" 75 } 76 }, 77 "checksum": 3610638499 78 }`, 79 "none", 80 containermap.ContainerMap{}, 81 "", 82 &stateMemory{ 83 assignments: ContainerCPUAssignments{ 84 "pod": map[string]cpuset.CPUSet{ 85 "container1": cpuset.NewCPUSet(4, 5, 6), 86 "container2": cpuset.NewCPUSet(1, 2, 3), 87 }, 88 }, 89 defaultCPUSet: cpuset.NewCPUSet(1, 2, 3), 90 }, 91 }, 92 { 93 "Restore checkpoint with invalid checksum", 94 `{ 95 "policyName": "none", 96 "defaultCPUSet": "4-6", 97 "entries": {}, 98 "checksum": 1337 99 }`, 100 "none", 101 containermap.ContainerMap{}, 102 "checkpoint is corrupted", 103 &stateMemory{}, 104 }, 105 { 106 "Restore checkpoint with invalid JSON", 107 `{`, 108 "none", 109 containermap.ContainerMap{}, 110 "unexpected end of JSON input", 111 &stateMemory{}, 112 }, 113 { 114 "Restore checkpoint with invalid policy name", 115 `{ 116 "policyName": "other", 117 "defaultCPUSet": "1-3", 118 "entries": {}, 119 "checksum": 1394507217 120 }`, 121 "none", 122 containermap.ContainerMap{}, 123 `configured policy "none" differs from state checkpoint policy "other"`, 124 &stateMemory{}, 125 }, 126 { 127 "Restore checkpoint with unparsable default cpu set", 128 `{ 129 "policyName": "none", 130 "defaultCPUSet": "1.3", 131 "entries": {}, 132 "checksum": 3021697696 133 }`, 134 "none", 135 containermap.ContainerMap{}, 136 `could not parse default cpu set "1.3": strconv.Atoi: parsing "1.3": invalid syntax`, 137 &stateMemory{}, 138 }, 139 { 140 "Restore checkpoint with unparsable assignment entry", 141 `{ 142 "policyName": "none", 143 "defaultCPUSet": "1-3", 144 "entries": { 145 "pod": { 146 "container1": "4-6", 147 "container2": "asd" 148 } 149 }, 150 "checksum": 962272150 151 }`, 152 "none", 153 containermap.ContainerMap{}, 154 `could not parse cpuset "asd" for container "container2" in pod "pod": strconv.Atoi: parsing "asd": invalid syntax`, 155 &stateMemory{}, 156 }, 157 { 158 "Restore checkpoint from checkpoint with v1 checksum", 159 `{ 160 "policyName": "none", 161 "defaultCPUSet": "1-3", 162 "checksum": 1694838852 163 }`, 164 "none", 165 containermap.ContainerMap{}, 166 "", 167 &stateMemory{ 168 defaultCPUSet: cpuset.NewCPUSet(1, 2, 3), 169 }, 170 }, 171 { 172 "Restore checkpoint with migration", 173 `{ 174 "policyName": "none", 175 "defaultCPUSet": "1-3", 176 "entries": { 177 "containerID1": "4-6", 178 "containerID2": "1-3" 179 }, 180 "checksum": 3680390589 181 }`, 182 "none", 183 func() containermap.ContainerMap { 184 cm := containermap.NewContainerMap() 185 cm.Add("pod", "container1", "containerID1") 186 cm.Add("pod", "container2", "containerID2") 187 return cm 188 }(), 189 "", 190 &stateMemory{ 191 assignments: ContainerCPUAssignments{ 192 "pod": map[string]cpuset.CPUSet{ 193 "container1": cpuset.NewCPUSet(4, 5, 6), 194 "container2": cpuset.NewCPUSet(1, 2, 3), 195 }, 196 }, 197 defaultCPUSet: cpuset.NewCPUSet(1, 2, 3), 198 }, 199 }, 200 } 201 202 // create temp dir 203 testingDir, err := ioutil.TempDir("", "cpumanager_state_test") 204 if err != nil { 205 t.Fatal(err) 206 } 207 defer os.RemoveAll(testingDir) 208 // create checkpoint manager for testing 209 cpm, err := checkpointmanager.NewCheckpointManager(testingDir) 210 if err != nil { 211 t.Fatalf("could not create testing checkpoint manager: %v", err) 212 } 213 214 for _, tc := range testCases { 215 t.Run(tc.description, func(t *testing.T) { 216 // ensure there is no previous checkpoint 217 cpm.RemoveCheckpoint(testingCheckpoint) 218 219 // prepare checkpoint for testing 220 if strings.TrimSpace(tc.checkpointContent) != "" { 221 checkpoint := &testutil.MockCheckpoint{Content: tc.checkpointContent} 222 if err := cpm.CreateCheckpoint(testingCheckpoint, checkpoint); err != nil { 223 t.Fatalf("could not create testing checkpoint: %v", err) 224 } 225 } 226 227 restoredState, err := NewCheckpointState(testingDir, testingCheckpoint, tc.policyName, tc.initialContainers) 228 if err != nil { 229 if strings.TrimSpace(tc.expectedError) != "" { 230 if strings.Contains(err.Error(), "could not restore state from checkpoint") && 231 strings.Contains(err.Error(), tc.expectedError) { 232 t.Logf("got expected error: %v", err) 233 return 234 } 235 } 236 t.Fatalf("unexpected error while creatng checkpointState: %v", err) 237 } 238 239 // compare state after restoration with the one expected 240 AssertStateEqual(t, restoredState, tc.expectedState) 241 }) 242 } 243} 244 245func TestCheckpointStateStore(t *testing.T) { 246 testCases := []struct { 247 description string 248 expectedState *stateMemory 249 }{ 250 { 251 "Store default cpu set", 252 &stateMemory{defaultCPUSet: cpuset.NewCPUSet(1, 2, 3)}, 253 }, 254 { 255 "Store assignments", 256 &stateMemory{ 257 assignments: map[string]map[string]cpuset.CPUSet{ 258 "pod": { 259 "container1": cpuset.NewCPUSet(1, 5, 8), 260 }, 261 }, 262 }, 263 }, 264 } 265 266 // create temp dir 267 testingDir, err := ioutil.TempDir("", "cpumanager_state_test") 268 if err != nil { 269 t.Fatal(err) 270 } 271 defer os.RemoveAll(testingDir) 272 273 cpm, err := checkpointmanager.NewCheckpointManager(testingDir) 274 if err != nil { 275 t.Fatalf("could not create testing checkpoint manager: %v", err) 276 } 277 278 for _, tc := range testCases { 279 t.Run(tc.description, func(t *testing.T) { 280 // ensure there is no previous checkpoint 281 cpm.RemoveCheckpoint(testingCheckpoint) 282 283 cs1, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil) 284 if err != nil { 285 t.Fatalf("could not create testing checkpointState instance: %v", err) 286 } 287 288 // set values of cs1 instance so they are stored in checkpoint and can be read by cs2 289 cs1.SetDefaultCPUSet(tc.expectedState.defaultCPUSet) 290 cs1.SetCPUAssignments(tc.expectedState.assignments) 291 292 // restore checkpoint with previously stored values 293 cs2, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil) 294 if err != nil { 295 t.Fatalf("could not create testing checkpointState instance: %v", err) 296 } 297 298 AssertStateEqual(t, cs2, tc.expectedState) 299 }) 300 } 301} 302 303func TestCheckpointStateHelpers(t *testing.T) { 304 testCases := []struct { 305 description string 306 defaultCPUset cpuset.CPUSet 307 assignments map[string]map[string]cpuset.CPUSet 308 }{ 309 { 310 description: "One container", 311 defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8), 312 assignments: map[string]map[string]cpuset.CPUSet{ 313 "pod": { 314 "c1": cpuset.NewCPUSet(0, 1), 315 }, 316 }, 317 }, 318 { 319 description: "Two containers", 320 defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8), 321 assignments: map[string]map[string]cpuset.CPUSet{ 322 "pod": { 323 "c1": cpuset.NewCPUSet(0, 1), 324 "c2": cpuset.NewCPUSet(2, 3, 4, 5), 325 }, 326 }, 327 }, 328 { 329 description: "Container without assigned cpus", 330 defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8), 331 assignments: map[string]map[string]cpuset.CPUSet{ 332 "pod": { 333 "c1": cpuset.NewCPUSet(), 334 }, 335 }, 336 }, 337 } 338 339 // create temp dir 340 testingDir, err := ioutil.TempDir("", "cpumanager_state_test") 341 if err != nil { 342 t.Fatal(err) 343 } 344 defer os.RemoveAll(testingDir) 345 346 cpm, err := checkpointmanager.NewCheckpointManager(testingDir) 347 if err != nil { 348 t.Fatalf("could not create testing checkpoint manager: %v", err) 349 } 350 351 for _, tc := range testCases { 352 t.Run(tc.description, func(t *testing.T) { 353 // ensure there is no previous checkpoint 354 cpm.RemoveCheckpoint(testingCheckpoint) 355 356 state, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil) 357 if err != nil { 358 t.Fatalf("could not create testing checkpointState instance: %v", err) 359 } 360 state.SetDefaultCPUSet(tc.defaultCPUset) 361 362 for pod := range tc.assignments { 363 for container, set := range tc.assignments[pod] { 364 state.SetCPUSet(pod, container, set) 365 if cpus, _ := state.GetCPUSet(pod, container); !cpus.Equals(set) { 366 t.Fatalf("state inconsistent, got %q instead of %q", set, cpus) 367 } 368 369 state.Delete(pod, container) 370 if _, ok := state.GetCPUSet(pod, container); ok { 371 t.Fatal("deleted container still existing in state") 372 } 373 } 374 } 375 }) 376 } 377} 378 379func TestCheckpointStateClear(t *testing.T) { 380 testCases := []struct { 381 description string 382 defaultCPUset cpuset.CPUSet 383 assignments map[string]map[string]cpuset.CPUSet 384 }{ 385 { 386 "Valid state", 387 cpuset.NewCPUSet(1, 5, 10), 388 map[string]map[string]cpuset.CPUSet{ 389 "pod": { 390 "container1": cpuset.NewCPUSet(1, 4), 391 }, 392 }, 393 }, 394 } 395 396 for _, tc := range testCases { 397 t.Run(tc.description, func(t *testing.T) { 398 // create temp dir 399 testingDir, err := ioutil.TempDir("", "cpumanager_state_test") 400 if err != nil { 401 t.Fatal(err) 402 } 403 defer os.RemoveAll(testingDir) 404 405 state, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil) 406 if err != nil { 407 t.Fatalf("could not create testing checkpointState instance: %v", err) 408 } 409 410 state.SetDefaultCPUSet(tc.defaultCPUset) 411 state.SetCPUAssignments(tc.assignments) 412 413 state.ClearState() 414 if !cpuset.NewCPUSet().Equals(state.GetDefaultCPUSet()) { 415 t.Fatal("cleared state with non-empty default cpu set") 416 } 417 for pod := range tc.assignments { 418 for container := range tc.assignments[pod] { 419 if _, ok := state.GetCPUSet(pod, container); ok { 420 t.Fatalf("container %q in pod %q with non-default cpu set in cleared state", container, pod) 421 } 422 } 423 } 424 }) 425 } 426} 427 428func AssertStateEqual(t *testing.T, sf State, sm State) { 429 cpusetSf := sf.GetDefaultCPUSet() 430 cpusetSm := sm.GetDefaultCPUSet() 431 if !cpusetSf.Equals(cpusetSm) { 432 t.Errorf("State CPUSet mismatch. Have %v, want %v", cpusetSf, cpusetSm) 433 } 434 435 cpuassignmentSf := sf.GetCPUAssignments() 436 cpuassignmentSm := sm.GetCPUAssignments() 437 if !reflect.DeepEqual(cpuassignmentSf, cpuassignmentSm) { 438 t.Errorf("State CPU assignments mismatch. Have %s, want %s", cpuassignmentSf, cpuassignmentSm) 439 } 440} 441