1/* 2Copyright 2019 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package topologymanager 18 19import ( 20 "fmt" 21 22 cadvisorapi "github.com/google/cadvisor/info/v1" 23 "k8s.io/api/core/v1" 24 "k8s.io/klog/v2" 25 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" 26 "k8s.io/kubernetes/pkg/kubelet/lifecycle" 27) 28 29const ( 30 // maxAllowableNUMANodes specifies the maximum number of NUMA Nodes that 31 // the TopologyManager supports on the underlying machine. 32 // 33 // At present, having more than this number of NUMA Nodes will result in a 34 // state explosion when trying to enumerate possible NUMAAffinity masks and 35 // generate hints for them. As such, if more NUMA Nodes than this are 36 // present on a machine and the TopologyManager is enabled, an error will 37 // be returned and the TopologyManager will not be loaded. 38 maxAllowableNUMANodes = 8 39 // ErrorTopologyAffinity represents the type for a TopologyAffinityError 40 ErrorTopologyAffinity = "TopologyAffinityError" 41) 42 43// TopologyAffinityError represents an resource alignment error 44type TopologyAffinityError struct{} 45 46func (e TopologyAffinityError) Error() string { 47 return "Resources cannot be allocated with Topology locality" 48} 49 50func (e TopologyAffinityError) Type() string { 51 return ErrorTopologyAffinity 52} 53 54// Manager interface provides methods for Kubelet to manage pod topology hints 55type Manager interface { 56 // PodAdmitHandler is implemented by Manager 57 lifecycle.PodAdmitHandler 58 // AddHintProvider adds a hint provider to manager to indicate the hint provider 59 // wants to be consulted with when making topology hints 60 AddHintProvider(HintProvider) 61 // AddContainer adds pod to Manager for tracking 62 AddContainer(pod *v1.Pod, container *v1.Container, containerID string) 63 // RemoveContainer removes pod from Manager tracking 64 RemoveContainer(containerID string) error 65 // Store is the interface for storing pod topology hints 66 Store 67} 68 69type manager struct { 70 //Topology Manager Scope 71 scope Scope 72} 73 74// HintProvider is an interface for components that want to collaborate to 75// achieve globally optimal concrete resource alignment with respect to 76// NUMA locality. 77type HintProvider interface { 78 // GetTopologyHints returns a map of resource names to a list of possible 79 // concrete resource allocations in terms of NUMA locality hints. Each hint 80 // is optionally marked "preferred" and indicates the set of NUMA nodes 81 // involved in the hypothetical allocation. The topology manager calls 82 // this function for each hint provider, and merges the hints to produce 83 // a consensus "best" hint. The hint providers may subsequently query the 84 // topology manager to influence actual resource assignment. 85 GetTopologyHints(pod *v1.Pod, container *v1.Container) map[string][]TopologyHint 86 // GetPodTopologyHints returns a map of resource names to a list of possible 87 // concrete resource allocations per Pod in terms of NUMA locality hints. 88 GetPodTopologyHints(pod *v1.Pod) map[string][]TopologyHint 89 // Allocate triggers resource allocation to occur on the HintProvider after 90 // all hints have been gathered and the aggregated Hint is available via a 91 // call to Store.GetAffinity(). 92 Allocate(pod *v1.Pod, container *v1.Container) error 93} 94 95//Store interface is to allow Hint Providers to retrieve pod affinity 96type Store interface { 97 GetAffinity(podUID string, containerName string) TopologyHint 98} 99 100// TopologyHint is a struct containing the NUMANodeAffinity for a Container 101type TopologyHint struct { 102 NUMANodeAffinity bitmask.BitMask 103 // Preferred is set to true when the NUMANodeAffinity encodes a preferred 104 // allocation for the Container. It is set to false otherwise. 105 Preferred bool 106} 107 108// IsEqual checks if TopologyHint are equal 109func (th *TopologyHint) IsEqual(topologyHint TopologyHint) bool { 110 if th.Preferred == topologyHint.Preferred { 111 if th.NUMANodeAffinity == nil || topologyHint.NUMANodeAffinity == nil { 112 return th.NUMANodeAffinity == topologyHint.NUMANodeAffinity 113 } 114 return th.NUMANodeAffinity.IsEqual(topologyHint.NUMANodeAffinity) 115 } 116 return false 117} 118 119// LessThan checks if TopologyHint `a` is less than TopologyHint `b` 120// this means that either `a` is a preferred hint and `b` is not 121// or `a` NUMANodeAffinity attribute is narrower than `b` NUMANodeAffinity attribute. 122func (th *TopologyHint) LessThan(other TopologyHint) bool { 123 if th.Preferred != other.Preferred { 124 return th.Preferred 125 } 126 return th.NUMANodeAffinity.IsNarrowerThan(other.NUMANodeAffinity) 127} 128 129var _ Manager = &manager{} 130 131// NewManager creates a new TopologyManager based on provided policy and scope 132func NewManager(topology []cadvisorapi.Node, topologyPolicyName string, topologyScopeName string) (Manager, error) { 133 klog.InfoS("Creating topology manager with policy per scope", "topologyPolicyName", topologyPolicyName, "topologyScopeName", topologyScopeName) 134 135 var numaNodes []int 136 for _, node := range topology { 137 numaNodes = append(numaNodes, node.Id) 138 } 139 140 if topologyPolicyName != PolicyNone && len(numaNodes) > maxAllowableNUMANodes { 141 return nil, fmt.Errorf("unsupported on machines with more than %v NUMA Nodes", maxAllowableNUMANodes) 142 } 143 144 var policy Policy 145 switch topologyPolicyName { 146 147 case PolicyNone: 148 policy = NewNonePolicy() 149 150 case PolicyBestEffort: 151 policy = NewBestEffortPolicy(numaNodes) 152 153 case PolicyRestricted: 154 policy = NewRestrictedPolicy(numaNodes) 155 156 case PolicySingleNumaNode: 157 policy = NewSingleNumaNodePolicy(numaNodes) 158 159 default: 160 return nil, fmt.Errorf("unknown policy: \"%s\"", topologyPolicyName) 161 } 162 163 var scope Scope 164 switch topologyScopeName { 165 166 case containerTopologyScope: 167 scope = NewContainerScope(policy) 168 169 case podTopologyScope: 170 scope = NewPodScope(policy) 171 172 default: 173 return nil, fmt.Errorf("unknown scope: \"%s\"", topologyScopeName) 174 } 175 176 manager := &manager{ 177 scope: scope, 178 } 179 180 return manager, nil 181} 182 183func (m *manager) GetAffinity(podUID string, containerName string) TopologyHint { 184 return m.scope.GetAffinity(podUID, containerName) 185} 186 187func (m *manager) AddHintProvider(h HintProvider) { 188 m.scope.AddHintProvider(h) 189} 190 191func (m *manager) AddContainer(pod *v1.Pod, container *v1.Container, containerID string) { 192 m.scope.AddContainer(pod, container, containerID) 193} 194 195func (m *manager) RemoveContainer(containerID string) error { 196 return m.scope.RemoveContainer(containerID) 197} 198 199func (m *manager) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAdmitResult { 200 klog.InfoS("Topology Admit Handler") 201 pod := attrs.Pod 202 203 return m.scope.Admit(pod) 204} 205