1/* 2Copyright 2019 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package podtopologyspread 18 19import ( 20 "context" 21 "testing" 22 23 "github.com/google/go-cmp/cmp" 24 appsv1 "k8s.io/api/apps/v1" 25 "k8s.io/api/core/v1" 26 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 "k8s.io/apimachinery/pkg/runtime" 28 "k8s.io/apimachinery/pkg/util/sets" 29 "k8s.io/client-go/informers" 30 "k8s.io/client-go/kubernetes/fake" 31 "k8s.io/kubernetes/pkg/scheduler/apis/config" 32 "k8s.io/kubernetes/pkg/scheduler/framework" 33 plugintesting "k8s.io/kubernetes/pkg/scheduler/framework/plugins/testing" 34 frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 35 "k8s.io/kubernetes/pkg/scheduler/internal/cache" 36 st "k8s.io/kubernetes/pkg/scheduler/testing" 37 "k8s.io/utils/pointer" 38) 39 40func TestPreScoreStateEmptyNodes(t *testing.T) { 41 tests := []struct { 42 name string 43 pod *v1.Pod 44 nodes []*v1.Node 45 objs []runtime.Object 46 config config.PodTopologySpreadArgs 47 want *preScoreState 48 }{ 49 { 50 name: "normal case", 51 pod: st.MakePod().Name("p").Label("foo", ""). 52 SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 53 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 54 Obj(), 55 nodes: []*v1.Node{ 56 st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), 57 st.MakeNode().Name("node-b").Label("zone", "zone1").Label(v1.LabelHostname, "node-b").Obj(), 58 st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), 59 }, 60 config: config.PodTopologySpreadArgs{ 61 DefaultingType: config.ListDefaulting, 62 }, 63 want: &preScoreState{ 64 Constraints: []topologySpreadConstraint{ 65 { 66 MaxSkew: 1, 67 TopologyKey: "zone", 68 Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), 69 }, 70 { 71 MaxSkew: 1, 72 TopologyKey: v1.LabelHostname, 73 Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), 74 }, 75 }, 76 IgnoredNodes: sets.NewString(), 77 TopologyPairToPodCounts: map[topologyPair]*int64{ 78 {key: "zone", value: "zone1"}: pointer.Int64Ptr(0), 79 {key: "zone", value: "zone2"}: pointer.Int64Ptr(0), 80 }, 81 TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2), topologyNormalizingWeight(3)}, 82 }, 83 }, 84 { 85 name: "node-x doesn't have label zone", 86 pod: st.MakePod().Name("p").Label("foo", ""). 87 SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 88 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). 89 Obj(), 90 nodes: []*v1.Node{ 91 st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), 92 st.MakeNode().Name("node-b").Label("zone", "zone1").Label(v1.LabelHostname, "node-b").Obj(), 93 st.MakeNode().Name("node-x").Label(v1.LabelHostname, "node-x").Obj(), 94 }, 95 config: config.PodTopologySpreadArgs{ 96 DefaultingType: config.ListDefaulting, 97 }, 98 want: &preScoreState{ 99 Constraints: []topologySpreadConstraint{ 100 { 101 MaxSkew: 1, 102 TopologyKey: "zone", 103 Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), 104 }, 105 { 106 MaxSkew: 1, 107 TopologyKey: v1.LabelHostname, 108 Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("bar").Obj()), 109 }, 110 }, 111 IgnoredNodes: sets.NewString("node-x"), 112 TopologyPairToPodCounts: map[topologyPair]*int64{ 113 {key: "zone", value: "zone1"}: pointer.Int64Ptr(0), 114 }, 115 TopologyNormalizingWeight: []float64{topologyNormalizingWeight(1), topologyNormalizingWeight(2)}, 116 }, 117 }, 118 { 119 name: "system default constraints and a replicaset", 120 pod: st.MakePod().Name("p").Namespace("default").Label("foo", "tar").Label("baz", "sup").OwnerReference("rs1", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")).Obj(), 121 config: config.PodTopologySpreadArgs{ 122 DefaultingType: config.SystemDefaulting, 123 }, 124 nodes: []*v1.Node{ 125 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Label(v1.LabelTopologyZone, "mars").Obj(), 126 }, 127 objs: []runtime.Object{ 128 &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("foo").Obj()}}, 129 }, 130 want: &preScoreState{ 131 Constraints: []topologySpreadConstraint{ 132 { 133 MaxSkew: 3, 134 TopologyKey: v1.LabelHostname, 135 Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), 136 }, 137 { 138 MaxSkew: 5, 139 TopologyKey: v1.LabelTopologyZone, 140 Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), 141 }, 142 }, 143 IgnoredNodes: sets.NewString(), 144 TopologyPairToPodCounts: map[topologyPair]*int64{ 145 {key: v1.LabelTopologyZone, value: "mars"}: pointer.Int64Ptr(0), 146 }, 147 TopologyNormalizingWeight: []float64{topologyNormalizingWeight(1), topologyNormalizingWeight(1)}, 148 }, 149 }, 150 { 151 name: "default constraints and a replicaset", 152 pod: st.MakePod().Name("p").Namespace("default").Label("foo", "tar").Label("baz", "sup").OwnerReference("rs1", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")).Obj(), 153 config: config.PodTopologySpreadArgs{ 154 DefaultConstraints: []v1.TopologySpreadConstraint{ 155 {MaxSkew: 1, TopologyKey: v1.LabelHostname, WhenUnsatisfiable: v1.ScheduleAnyway}, 156 {MaxSkew: 2, TopologyKey: "rack", WhenUnsatisfiable: v1.DoNotSchedule}, 157 {MaxSkew: 2, TopologyKey: "planet", WhenUnsatisfiable: v1.ScheduleAnyway}, 158 }, 159 DefaultingType: config.ListDefaulting, 160 }, 161 nodes: []*v1.Node{ 162 st.MakeNode().Name("node-a").Label("rack", "rack1").Label(v1.LabelHostname, "node-a").Label("planet", "mars").Obj(), 163 }, 164 objs: []runtime.Object{ 165 &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("foo").Obj()}}, 166 }, 167 want: &preScoreState{ 168 Constraints: []topologySpreadConstraint{ 169 { 170 MaxSkew: 1, 171 TopologyKey: v1.LabelHostname, 172 Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), 173 }, 174 { 175 MaxSkew: 2, 176 TopologyKey: "planet", 177 Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), 178 }, 179 }, 180 IgnoredNodes: sets.NewString(), 181 TopologyPairToPodCounts: map[topologyPair]*int64{ 182 {key: "planet", value: "mars"}: pointer.Int64Ptr(0), 183 }, 184 TopologyNormalizingWeight: []float64{topologyNormalizingWeight(1), topologyNormalizingWeight(1)}, 185 }, 186 }, 187 { 188 name: "default constraints and a replicaset that doesn't match", 189 pod: st.MakePod().Name("p").Namespace("default").Label("foo", "bar").Label("baz", "sup").OwnerReference("rs2", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")).Obj(), 190 config: config.PodTopologySpreadArgs{ 191 DefaultConstraints: []v1.TopologySpreadConstraint{ 192 {MaxSkew: 2, TopologyKey: "planet", WhenUnsatisfiable: v1.ScheduleAnyway}, 193 }, 194 DefaultingType: config.ListDefaulting, 195 }, 196 nodes: []*v1.Node{ 197 st.MakeNode().Name("node-a").Label("planet", "mars").Obj(), 198 }, 199 objs: []runtime.Object{ 200 &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("tar").Obj()}}, 201 }, 202 want: &preScoreState{ 203 TopologyPairToPodCounts: make(map[topologyPair]*int64), 204 }, 205 }, 206 { 207 name: "default constraints and a replicaset, but pod has constraints", 208 pod: st.MakePod().Name("p").Namespace("default").Label("foo", "bar").Label("baz", "sup"). 209 OwnerReference("rs1", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")). 210 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("foo", "bar").Obj()). 211 SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("baz", "sup").Obj()).Obj(), 212 config: config.PodTopologySpreadArgs{ 213 DefaultConstraints: []v1.TopologySpreadConstraint{ 214 {MaxSkew: 2, TopologyKey: "galaxy", WhenUnsatisfiable: v1.ScheduleAnyway}, 215 }, 216 DefaultingType: config.ListDefaulting, 217 }, 218 nodes: []*v1.Node{ 219 st.MakeNode().Name("node-a").Label("planet", "mars").Label("galaxy", "andromeda").Obj(), 220 }, 221 objs: []runtime.Object{ 222 &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("foo").Obj()}}, 223 }, 224 want: &preScoreState{ 225 Constraints: []topologySpreadConstraint{ 226 { 227 MaxSkew: 2, 228 TopologyKey: "planet", 229 Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("baz", "sup").Obj()), 230 }, 231 }, 232 IgnoredNodes: sets.NewString(), 233 TopologyPairToPodCounts: map[topologyPair]*int64{ 234 {"planet", "mars"}: pointer.Int64Ptr(0), 235 }, 236 TopologyNormalizingWeight: []float64{topologyNormalizingWeight(1)}, 237 }, 238 }, 239 } 240 for _, tt := range tests { 241 t.Run(tt.name, func(t *testing.T) { 242 ctx := context.Background() 243 informerFactory := informers.NewSharedInformerFactory(fake.NewSimpleClientset(tt.objs...), 0) 244 f, err := frameworkruntime.NewFramework(nil, nil, 245 frameworkruntime.WithSnapshotSharedLister(cache.NewSnapshot(nil, tt.nodes)), 246 frameworkruntime.WithInformerFactory(informerFactory)) 247 if err != nil { 248 t.Fatalf("Failed creating framework runtime: %v", err) 249 } 250 pl, err := New(&tt.config, f) 251 if err != nil { 252 t.Fatalf("Failed creating plugin: %v", err) 253 } 254 informerFactory.Start(ctx.Done()) 255 informerFactory.WaitForCacheSync(ctx.Done()) 256 p := pl.(*PodTopologySpread) 257 cs := framework.NewCycleState() 258 if s := p.PreScore(context.Background(), cs, tt.pod, tt.nodes); !s.IsSuccess() { 259 t.Fatal(s.AsError()) 260 } 261 262 got, err := getPreScoreState(cs) 263 if err != nil { 264 t.Fatal(err) 265 } 266 if diff := cmp.Diff(tt.want, got, cmpOpts...); diff != "" { 267 t.Errorf("PodTopologySpread#PreScore() returned (-want, +got):\n%s", diff) 268 } 269 }) 270 } 271} 272 273func TestPodTopologySpreadScore(t *testing.T) { 274 tests := []struct { 275 name string 276 pod *v1.Pod 277 existingPods []*v1.Pod 278 nodes []*v1.Node 279 failedNodes []*v1.Node // nodes + failedNodes = all nodes 280 want framework.NodeScoreList 281 }{ 282 // Explanation on the Legend: 283 // a) X/Y means there are X matching pods on node1 and Y on node2, both nodes are candidates 284 // (i.e. they have passed all predicates) 285 // b) X/~Y~ means there are X matching pods on node1 and Y on node2, but node Y is NOT a candidate 286 // c) X/?Y? means there are X matching pods on node1 and Y on node2, both nodes are candidates 287 // but node2 either i) doesn't have all required topologyKeys present, or ii) doesn't match 288 // incoming pod's nodeSelector/nodeAffinity 289 { 290 name: "one constraint on node, no existing pods", 291 pod: st.MakePod().Name("p").Label("foo", ""). 292 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 293 Obj(), 294 nodes: []*v1.Node{ 295 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 296 st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), 297 }, 298 want: []framework.NodeScore{ 299 {Name: "node-a", Score: 100}, 300 {Name: "node-b", Score: 100}, 301 }, 302 }, 303 { 304 // if there is only one candidate node, it should be scored to 100 305 name: "one constraint on node, only one node is candidate", 306 pod: st.MakePod().Name("p").Label("foo", ""). 307 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 308 Obj(), 309 existingPods: []*v1.Pod{ 310 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 311 st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), 312 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 313 }, 314 nodes: []*v1.Node{ 315 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 316 }, 317 failedNodes: []*v1.Node{ 318 st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), 319 }, 320 want: []framework.NodeScore{ 321 {Name: "node-a", Score: 100}, 322 }, 323 }, 324 { 325 name: "one constraint on node, all nodes have the same number of matching pods", 326 pod: st.MakePod().Name("p").Label("foo", ""). 327 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 328 Obj(), 329 existingPods: []*v1.Pod{ 330 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 331 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 332 }, 333 nodes: []*v1.Node{ 334 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 335 st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), 336 }, 337 want: []framework.NodeScore{ 338 {Name: "node-a", Score: 100}, 339 {Name: "node-b", Score: 100}, 340 }, 341 }, 342 { 343 // matching pods spread as 2/1/0/3. 344 name: "one constraint on node, all 4 nodes are candidates", 345 pod: st.MakePod().Name("p").Label("foo", ""). 346 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 347 Obj(), 348 existingPods: []*v1.Pod{ 349 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 350 st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), 351 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 352 st.MakePod().Name("p-d1").Node("node-d").Label("foo", "").Obj(), 353 st.MakePod().Name("p-d2").Node("node-d").Label("foo", "").Obj(), 354 st.MakePod().Name("p-d3").Node("node-d").Label("foo", "").Obj(), 355 }, 356 nodes: []*v1.Node{ 357 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 358 st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), 359 st.MakeNode().Name("node-c").Label(v1.LabelHostname, "node-c").Obj(), 360 st.MakeNode().Name("node-d").Label(v1.LabelHostname, "node-d").Obj(), 361 }, 362 failedNodes: []*v1.Node{}, 363 want: []framework.NodeScore{ 364 {Name: "node-a", Score: 40}, 365 {Name: "node-b", Score: 80}, 366 {Name: "node-c", Score: 100}, 367 {Name: "node-d", Score: 0}, 368 }, 369 }, 370 { 371 name: "one constraint on node, all 4 nodes are candidates, maxSkew=2", 372 pod: st.MakePod().Name("p").Label("foo", ""). 373 SpreadConstraint(2, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 374 Obj(), 375 // matching pods spread as 2/1/0/3. 376 existingPods: []*v1.Pod{ 377 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 378 st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), 379 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 380 st.MakePod().Name("p-d1").Node("node-d").Label("foo", "").Obj(), 381 st.MakePod().Name("p-d2").Node("node-d").Label("foo", "").Obj(), 382 st.MakePod().Name("p-d3").Node("node-d").Label("foo", "").Obj(), 383 }, 384 nodes: []*v1.Node{ 385 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 386 st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), 387 st.MakeNode().Name("node-c").Label(v1.LabelHostname, "node-c").Obj(), 388 st.MakeNode().Name("node-d").Label(v1.LabelHostname, "node-d").Obj(), 389 }, 390 failedNodes: []*v1.Node{}, 391 want: []framework.NodeScore{ 392 {Name: "node-a", Score: 50}, // +10, compared to maxSkew=1 393 {Name: "node-b", Score: 83}, // +3, compared to maxSkew=1 394 {Name: "node-c", Score: 100}, 395 {Name: "node-d", Score: 16}, // +16, compared to maxSkew=1 396 }, 397 }, 398 { 399 name: "one constraint on node, all 4 nodes are candidates, maxSkew=3", 400 pod: st.MakePod().Name("p").Label("foo", ""). 401 SpreadConstraint(3, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 402 Obj(), 403 existingPods: []*v1.Pod{ 404 // matching pods spread as 4/3/2/1. 405 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 406 st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), 407 st.MakePod().Name("p-a3").Node("node-a").Label("foo", "").Obj(), 408 st.MakePod().Name("p-a4").Node("node-a").Label("foo", "").Obj(), 409 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 410 st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), 411 st.MakePod().Name("p-b3").Node("node-b").Label("foo", "").Obj(), 412 st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), 413 st.MakePod().Name("p-c2").Node("node-c").Label("foo", "").Obj(), 414 st.MakePod().Name("p-d1").Node("node-d").Label("foo", "").Obj(), 415 }, 416 nodes: []*v1.Node{ 417 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 418 st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), 419 st.MakeNode().Name("node-c").Label(v1.LabelHostname, "node-c").Obj(), 420 st.MakeNode().Name("node-d").Label(v1.LabelHostname, "node-d").Obj(), 421 }, 422 failedNodes: []*v1.Node{}, 423 want: []framework.NodeScore{ 424 {Name: "node-a", Score: 33}, // +19 compared to maxSkew=1 425 {Name: "node-b", Score: 55}, // +13 compared to maxSkew=1 426 {Name: "node-c", Score: 77}, // +6 compared to maxSkew=1 427 {Name: "node-d", Score: 100}, 428 }, 429 }, 430 { 431 // matching pods spread as 4/2/1/~3~ (node4 is not a candidate) 432 name: "one constraint on node, 3 out of 4 nodes are candidates", 433 pod: st.MakePod().Name("p").Label("foo", ""). 434 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 435 Obj(), 436 existingPods: []*v1.Pod{ 437 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 438 st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), 439 st.MakePod().Name("p-a3").Node("node-a").Label("foo", "").Obj(), 440 st.MakePod().Name("p-a4").Node("node-a").Label("foo", "").Obj(), 441 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 442 st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), 443 st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(), 444 st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), 445 st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(), 446 st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(), 447 }, 448 nodes: []*v1.Node{ 449 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 450 st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), 451 st.MakeNode().Name("node-x").Label(v1.LabelHostname, "node-x").Obj(), 452 }, 453 failedNodes: []*v1.Node{ 454 st.MakeNode().Name("node-y").Label(v1.LabelHostname, "node-y").Obj(), 455 }, 456 want: []framework.NodeScore{ 457 {Name: "node-a", Score: 16}, 458 {Name: "node-b", Score: 66}, 459 {Name: "node-x", Score: 100}, 460 }, 461 }, 462 { 463 // matching pods spread as 4/?2?/1/~3~, total = 4+?+1 = 5 (as node2 is problematic) 464 name: "one constraint on node, 3 out of 4 nodes are candidates, one node doesn't match topology key", 465 pod: st.MakePod().Name("p").Label("foo", ""). 466 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 467 Obj(), 468 existingPods: []*v1.Pod{ 469 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 470 st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), 471 st.MakePod().Name("p-a3").Node("node-a").Label("foo", "").Obj(), 472 st.MakePod().Name("p-a4").Node("node-a").Label("foo", "").Obj(), 473 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 474 st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), 475 st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(), 476 st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), 477 st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(), 478 st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(), 479 }, 480 nodes: []*v1.Node{ 481 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 482 st.MakeNode().Name("node-b").Label("n", "node-b").Obj(), // label `n` doesn't match topologyKey 483 st.MakeNode().Name("node-x").Label(v1.LabelHostname, "node-x").Obj(), 484 }, 485 failedNodes: []*v1.Node{ 486 st.MakeNode().Name("node-y").Label(v1.LabelHostname, "node-y").Obj(), 487 }, 488 want: []framework.NodeScore{ 489 {Name: "node-a", Score: 20}, 490 {Name: "node-b", Score: 0}, 491 {Name: "node-x", Score: 100}, 492 }, 493 }, 494 { 495 // matching pods spread as 4/2/1/~3~ 496 name: "one constraint on zone, 3 out of 4 nodes are candidates", 497 pod: st.MakePod().Name("p").Label("foo", ""). 498 SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 499 Obj(), 500 existingPods: []*v1.Pod{ 501 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 502 st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), 503 st.MakePod().Name("p-a3").Node("node-a").Label("foo", "").Obj(), 504 st.MakePod().Name("p-a4").Node("node-a").Label("foo", "").Obj(), 505 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 506 st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), 507 st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(), 508 st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), 509 st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(), 510 st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(), 511 }, 512 nodes: []*v1.Node{ 513 st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), 514 st.MakeNode().Name("node-b").Label("zone", "zone1").Label(v1.LabelHostname, "node-b").Obj(), 515 st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), 516 }, 517 failedNodes: []*v1.Node{ 518 st.MakeNode().Name("node-y").Label("zone", "zone2").Label(v1.LabelHostname, "node-y").Obj(), 519 }, 520 want: []framework.NodeScore{ 521 {Name: "node-a", Score: 62}, 522 {Name: "node-b", Score: 62}, 523 {Name: "node-x", Score: 100}, 524 }, 525 }, 526 { 527 // matching pods spread as 2/~1~/2/~4~. 528 name: "two Constraints on zone and node, 2 out of 4 nodes are candidates", 529 pod: st.MakePod().Name("p").Label("foo", ""). 530 SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 531 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 532 Obj(), 533 existingPods: []*v1.Pod{ 534 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 535 st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), 536 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 537 st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(), 538 st.MakePod().Name("p-x2").Node("node-x").Label("foo", "").Obj(), 539 st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), 540 st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(), 541 st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(), 542 st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(), 543 }, 544 nodes: []*v1.Node{ 545 st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), 546 st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), 547 }, 548 failedNodes: []*v1.Node{ 549 st.MakeNode().Name("node-b").Label("zone", "zone1").Label(v1.LabelHostname, "node-b").Obj(), 550 st.MakeNode().Name("node-y").Label("zone", "zone2").Label(v1.LabelHostname, "node-y").Obj(), 551 }, 552 want: []framework.NodeScore{ 553 {Name: "node-a", Score: 100}, 554 {Name: "node-x", Score: 54}, 555 }, 556 }, 557 { 558 // If Constraints hold different labelSelectors, it's a little complex. 559 // +----------------------+------------------------+ 560 // | zone1 | zone2 | 561 // +----------------------+------------------------+ 562 // | node-a | node-b | node-x | node-y | 563 // +--------+-------------+--------+---------------+ 564 // | P{foo} | P{foo, bar} | | P{foo} P{bar} | 565 // +--------+-------------+--------+---------------+ 566 // For the first constraint (zone): the matching pods spread as 2/2/1/1 567 // For the second constraint (node): the matching pods spread as 0/1/0/1 568 name: "two Constraints on zone and node, with different labelSelectors", 569 pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). 570 SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 571 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). 572 Obj(), 573 existingPods: []*v1.Pod{ 574 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 575 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Label("bar", "").Obj(), 576 st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), 577 st.MakePod().Name("p-y2").Node("node-y").Label("bar", "").Obj(), 578 }, 579 nodes: []*v1.Node{ 580 st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), 581 st.MakeNode().Name("node-b").Label("zone", "zone1").Label(v1.LabelHostname, "node-b").Obj(), 582 st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), 583 st.MakeNode().Name("node-y").Label("zone", "zone2").Label(v1.LabelHostname, "node-y").Obj(), 584 }, 585 failedNodes: []*v1.Node{}, 586 want: []framework.NodeScore{ 587 {Name: "node-a", Score: 75}, 588 {Name: "node-b", Score: 25}, 589 {Name: "node-x", Score: 100}, 590 {Name: "node-y", Score: 50}, 591 }, 592 }, 593 { 594 // For the first constraint (zone): the matching pods spread as 0/0/2/2 595 // For the second constraint (node): the matching pods spread as 0/1/0/1 596 name: "two Constraints on zone and node, with different labelSelectors, some nodes have 0 pods", 597 pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). 598 SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 599 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). 600 Obj(), 601 existingPods: []*v1.Pod{ 602 st.MakePod().Name("p-b1").Node("node-b").Label("bar", "").Obj(), 603 st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(), 604 st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Label("bar", "").Obj(), 605 }, 606 nodes: []*v1.Node{ 607 st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), 608 st.MakeNode().Name("node-b").Label("zone", "zone1").Label(v1.LabelHostname, "node-b").Obj(), 609 st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), 610 st.MakeNode().Name("node-y").Label("zone", "zone2").Label(v1.LabelHostname, "node-y").Obj(), 611 }, 612 failedNodes: []*v1.Node{}, 613 want: []framework.NodeScore{ 614 {Name: "node-a", Score: 100}, 615 {Name: "node-b", Score: 75}, 616 {Name: "node-x", Score: 50}, 617 {Name: "node-y", Score: 0}, 618 }, 619 }, 620 { 621 // For the first constraint (zone): the matching pods spread as 2/2/1/~1~ 622 // For the second constraint (node): the matching pods spread as 0/1/0/~1~ 623 name: "two Constraints on zone and node, with different labelSelectors, 3 out of 4 nodes are candidates", 624 pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). 625 SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 626 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). 627 Obj(), 628 existingPods: []*v1.Pod{ 629 st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), 630 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Label("bar", "").Obj(), 631 st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), 632 st.MakePod().Name("p-y2").Node("node-y").Label("bar", "").Obj(), 633 }, 634 nodes: []*v1.Node{ 635 st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), 636 st.MakeNode().Name("node-b").Label("zone", "zone1").Label(v1.LabelHostname, "node-b").Obj(), 637 st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), 638 }, 639 failedNodes: []*v1.Node{ 640 st.MakeNode().Name("node-y").Label("zone", "zone2").Label(v1.LabelHostname, "node-y").Obj(), 641 }, 642 want: []framework.NodeScore{ 643 {Name: "node-a", Score: 75}, 644 {Name: "node-b", Score: 25}, 645 {Name: "node-x", Score: 100}, 646 }, 647 }, 648 { 649 name: "existing pods in a different namespace do not count", 650 pod: st.MakePod().Name("p").Label("foo", ""). 651 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 652 Obj(), 653 existingPods: []*v1.Pod{ 654 st.MakePod().Name("p-a1").Namespace("ns1").Node("node-a").Label("foo", "").Obj(), 655 st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), 656 st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), 657 st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), 658 }, 659 nodes: []*v1.Node{ 660 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 661 st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), 662 }, 663 want: []framework.NodeScore{ 664 {Name: "node-a", Score: 100}, 665 {Name: "node-b", Score: 50}, 666 }, 667 }, 668 { 669 name: "terminating Pods should be excluded", 670 pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( 671 1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), 672 ).Obj(), 673 nodes: []*v1.Node{ 674 st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), 675 st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), 676 }, 677 existingPods: []*v1.Pod{ 678 st.MakePod().Name("p-a").Node("node-a").Label("foo", "").Terminating().Obj(), 679 st.MakePod().Name("p-b").Node("node-b").Label("foo", "").Obj(), 680 }, 681 want: []framework.NodeScore{ 682 {Name: "node-a", Score: 100}, 683 {Name: "node-b", Score: 0}, 684 }, 685 }, 686 } 687 for _, tt := range tests { 688 t.Run(tt.name, func(t *testing.T) { 689 allNodes := append([]*v1.Node{}, tt.nodes...) 690 allNodes = append(allNodes, tt.failedNodes...) 691 state := framework.NewCycleState() 692 pl := plugintesting.SetupPlugin(t, New, &config.PodTopologySpreadArgs{DefaultingType: config.ListDefaulting}, cache.NewSnapshot(tt.existingPods, allNodes)) 693 p := pl.(*PodTopologySpread) 694 695 status := p.PreScore(context.Background(), state, tt.pod, tt.nodes) 696 if !status.IsSuccess() { 697 t.Errorf("unexpected error: %v", status) 698 } 699 700 var gotList framework.NodeScoreList 701 for _, n := range tt.nodes { 702 nodeName := n.Name 703 score, status := p.Score(context.Background(), state, tt.pod, nodeName) 704 if !status.IsSuccess() { 705 t.Errorf("unexpected error: %v", status) 706 } 707 gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score}) 708 } 709 710 status = p.NormalizeScore(context.Background(), state, tt.pod, gotList) 711 if !status.IsSuccess() { 712 t.Errorf("unexpected error: %v", status) 713 } 714 if diff := cmp.Diff(tt.want, gotList, cmpOpts...); diff != "" { 715 t.Errorf("unexpected scores (-want,+got):\n%s", diff) 716 } 717 }) 718 } 719} 720 721func BenchmarkTestPodTopologySpreadScore(b *testing.B) { 722 tests := []struct { 723 name string 724 pod *v1.Pod 725 existingPodsNum int 726 allNodesNum int 727 filteredNodesNum int 728 }{ 729 { 730 name: "1000nodes/single-constraint-zone", 731 pod: st.MakePod().Name("p").Label("foo", ""). 732 SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 733 Obj(), 734 existingPodsNum: 10000, 735 allNodesNum: 1000, 736 filteredNodesNum: 500, 737 }, 738 { 739 name: "1000nodes/single-constraint-node", 740 pod: st.MakePod().Name("p").Label("foo", ""). 741 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 742 Obj(), 743 existingPodsNum: 10000, 744 allNodesNum: 1000, 745 filteredNodesNum: 500, 746 }, 747 { 748 name: "1000nodes/two-Constraints-zone-node", 749 pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). 750 SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). 751 SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). 752 Obj(), 753 existingPodsNum: 10000, 754 allNodesNum: 1000, 755 filteredNodesNum: 500, 756 }, 757 } 758 for _, tt := range tests { 759 b.Run(tt.name, func(b *testing.B) { 760 existingPods, allNodes, filteredNodes := st.MakeNodesAndPodsForEvenPodsSpread(tt.pod.Labels, tt.existingPodsNum, tt.allNodesNum, tt.filteredNodesNum) 761 state := framework.NewCycleState() 762 pl := plugintesting.SetupPlugin(b, New, &config.PodTopologySpreadArgs{DefaultingType: config.ListDefaulting}, cache.NewSnapshot(existingPods, allNodes)) 763 p := pl.(*PodTopologySpread) 764 765 status := p.PreScore(context.Background(), state, tt.pod, filteredNodes) 766 if !status.IsSuccess() { 767 b.Fatalf("unexpected error: %v", status) 768 } 769 b.ResetTimer() 770 771 for i := 0; i < b.N; i++ { 772 var gotList framework.NodeScoreList 773 for _, n := range filteredNodes { 774 nodeName := n.Name 775 score, status := p.Score(context.Background(), state, tt.pod, nodeName) 776 if !status.IsSuccess() { 777 b.Fatalf("unexpected error: %v", status) 778 } 779 gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score}) 780 } 781 782 status = p.NormalizeScore(context.Background(), state, tt.pod, gotList) 783 if !status.IsSuccess() { 784 b.Fatal(status) 785 } 786 } 787 }) 788 } 789} 790 791// The following test allows to compare PodTopologySpread.Score with 792// SelectorSpread.Score by using a similar rule. 793// See pkg/scheduler/framework/plugins/selectorspread/selector_spread_perf_test.go 794// for the equivalent test. 795 796var ( 797 tests = []struct { 798 name string 799 existingPodsNum int 800 allNodesNum int 801 }{ 802 { 803 name: "100nodes", 804 existingPodsNum: 1000, 805 allNodesNum: 100, 806 }, 807 { 808 name: "1000nodes", 809 existingPodsNum: 10000, 810 allNodesNum: 1000, 811 }, 812 { 813 name: "5000nodes", 814 existingPodsNum: 50000, 815 allNodesNum: 5000, 816 }, 817 } 818) 819 820func BenchmarkTestDefaultEvenPodsSpreadPriority(b *testing.B) { 821 for _, tt := range tests { 822 b.Run(tt.name, func(b *testing.B) { 823 pod := st.MakePod().Name("p").Label("foo", "").Obj() 824 existingPods, allNodes, filteredNodes := st.MakeNodesAndPodsForEvenPodsSpread(pod.Labels, tt.existingPodsNum, tt.allNodesNum, tt.allNodesNum) 825 state := framework.NewCycleState() 826 snapshot := cache.NewSnapshot(existingPods, allNodes) 827 client := fake.NewSimpleClientset( 828 &v1.Service{Spec: v1.ServiceSpec{Selector: map[string]string{"foo": ""}}}, 829 ) 830 ctx := context.Background() 831 informerFactory := informers.NewSharedInformerFactory(client, 0) 832 f, err := frameworkruntime.NewFramework(nil, nil, 833 frameworkruntime.WithSnapshotSharedLister(snapshot), 834 frameworkruntime.WithInformerFactory(informerFactory)) 835 if err != nil { 836 b.Fatalf("Failed creating framework runtime: %v", err) 837 } 838 pl, err := New(&config.PodTopologySpreadArgs{DefaultingType: config.SystemDefaulting}, f) 839 if err != nil { 840 b.Fatalf("Failed creating plugin: %v", err) 841 } 842 p := pl.(*PodTopologySpread) 843 844 informerFactory.Start(ctx.Done()) 845 informerFactory.WaitForCacheSync(ctx.Done()) 846 b.ResetTimer() 847 848 for i := 0; i < b.N; i++ { 849 status := p.PreScore(ctx, state, pod, filteredNodes) 850 if !status.IsSuccess() { 851 b.Fatalf("unexpected error: %v", status) 852 } 853 gotList := make(framework.NodeScoreList, len(filteredNodes)) 854 scoreNode := func(i int) { 855 n := filteredNodes[i] 856 score, _ := p.Score(ctx, state, pod, n.Name) 857 gotList[i] = framework.NodeScore{Name: n.Name, Score: score} 858 } 859 p.parallelizer.Until(ctx, len(filteredNodes), scoreNode) 860 status = p.NormalizeScore(ctx, state, pod, gotList) 861 if !status.IsSuccess() { 862 b.Fatal(status) 863 } 864 } 865 }) 866 } 867} 868