1/* 2Copyright 2017 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package dns 18 19import ( 20 "context" 21 "fmt" 22 "strings" 23 24 kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" 25 kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" 26 "k8s.io/kubernetes/cmd/kubeadm/app/features" 27 "k8s.io/kubernetes/cmd/kubeadm/app/images" 28 kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" 29 "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" 30 31 apps "k8s.io/api/apps/v1" 32 v1 "k8s.io/api/core/v1" 33 rbac "k8s.io/api/rbac/v1" 34 apierrors "k8s.io/apimachinery/pkg/api/errors" 35 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 36 kuberuntime "k8s.io/apimachinery/pkg/runtime" 37 "k8s.io/apimachinery/pkg/types" 38 clientset "k8s.io/client-go/kubernetes" 39 clientsetscheme "k8s.io/client-go/kubernetes/scheme" 40 "k8s.io/klog/v2" 41 42 "github.com/coredns/corefile-migration/migration" 43 "github.com/pkg/errors" 44) 45 46const ( 47 unableToDecodeCoreDNS = "unable to decode CoreDNS" 48 coreDNSReplicas = 2 49) 50 51// DeployedDNSAddon returns the type of DNS addon currently deployed 52func DeployedDNSAddon(client clientset.Interface) (string, error) { 53 deploymentsClient := client.AppsV1().Deployments(metav1.NamespaceSystem) 54 deployments, err := deploymentsClient.List(context.TODO(), metav1.ListOptions{LabelSelector: "k8s-app=kube-dns"}) 55 if err != nil { 56 return "", errors.Wrap(err, "couldn't retrieve DNS addon deployments") 57 } 58 59 switch len(deployments.Items) { 60 case 0: 61 return "", nil 62 case 1: 63 addonImage := deployments.Items[0].Spec.Template.Spec.Containers[0].Image 64 addonImageParts := strings.Split(addonImage, ":") 65 addonVersion := addonImageParts[len(addonImageParts)-1] 66 return addonVersion, nil 67 default: 68 return "", errors.Errorf("multiple DNS addon deployments found: %v", deployments.Items) 69 } 70} 71 72// deployedDNSReplicas returns the replica count for the current DNS deployment 73func deployedDNSReplicas(client clientset.Interface, replicas int32) (*int32, error) { 74 deploymentsClient := client.AppsV1().Deployments(metav1.NamespaceSystem) 75 deployments, err := deploymentsClient.List(context.TODO(), metav1.ListOptions{LabelSelector: "k8s-app=kube-dns"}) 76 if err != nil { 77 return &replicas, errors.Wrap(err, "couldn't retrieve DNS addon deployments") 78 } 79 switch len(deployments.Items) { 80 case 0: 81 return &replicas, nil 82 case 1: 83 return deployments.Items[0].Spec.Replicas, nil 84 default: 85 return &replicas, errors.Errorf("multiple DNS addon deployments found: %v", deployments.Items) 86 } 87} 88 89// EnsureDNSAddon creates the CoreDNS addon 90func EnsureDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error { 91 replicas, err := deployedDNSReplicas(client, coreDNSReplicas) 92 if err != nil { 93 return err 94 } 95 return coreDNSAddon(cfg, client, replicas) 96} 97 98func coreDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, replicas *int32) error { 99 // Get the YAML manifest 100 coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct { 101 DeploymentName, Image, OldControlPlaneTaintKey, ControlPlaneTaintKey string 102 Replicas *int32 103 }{ 104 DeploymentName: kubeadmconstants.CoreDNSDeploymentName, 105 Image: images.GetDNSImage(cfg), 106 // TODO: https://github.com/kubernetes/kubeadm/issues/2200 107 OldControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleOldControlPlane, 108 ControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleControlPlane, 109 Replicas: replicas, 110 }) 111 if err != nil { 112 return errors.Wrap(err, "error when parsing CoreDNS deployment template") 113 } 114 115 // Get the config file for CoreDNS 116 coreDNSConfigMapBytes, err := kubeadmutil.ParseTemplate(CoreDNSConfigMap, struct{ DNSDomain, UpstreamNameserver, StubDomain string }{ 117 DNSDomain: cfg.Networking.DNSDomain, 118 }) 119 if err != nil { 120 return errors.Wrap(err, "error when parsing CoreDNS configMap template") 121 } 122 123 dnsip, err := kubeadmconstants.GetDNSIP(cfg.Networking.ServiceSubnet, features.Enabled(cfg.FeatureGates, features.IPv6DualStack)) 124 if err != nil { 125 return err 126 } 127 128 coreDNSServiceBytes, err := kubeadmutil.ParseTemplate(CoreDNSService, struct{ DNSIP string }{ 129 DNSIP: dnsip.String(), 130 }) 131 132 if err != nil { 133 return errors.Wrap(err, "error when parsing CoreDNS service template") 134 } 135 136 if err := createCoreDNSAddon(coreDNSDeploymentBytes, coreDNSServiceBytes, coreDNSConfigMapBytes, client); err != nil { 137 return err 138 } 139 fmt.Println("[addons] Applied essential addon: CoreDNS") 140 return nil 141} 142 143func createCoreDNSAddon(deploymentBytes, serviceBytes, configBytes []byte, client clientset.Interface) error { 144 coreDNSConfigMap := &v1.ConfigMap{} 145 if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), configBytes, coreDNSConfigMap); err != nil { 146 return errors.Wrapf(err, "%s ConfigMap", unableToDecodeCoreDNS) 147 } 148 149 // Create the ConfigMap for CoreDNS or update/migrate it in case it already exists 150 _, corefile, currentInstalledCoreDNSVersion, err := GetCoreDNSInfo(client) 151 if err != nil { 152 return errors.Wrap(err, "unable to fetch CoreDNS current installed version and ConfigMap.") 153 } 154 155 corefileMigrationRequired, err := isCoreDNSConfigMapMigrationRequired(corefile, currentInstalledCoreDNSVersion) 156 if err != nil { 157 return err 158 } 159 160 // Assume that migration is always possible, rely on migrateCoreDNSCorefile() to fail if not. 161 canMigrateCorefile := true 162 163 if corefile == "" || migration.Default("", corefile) { 164 // If the Corefile is empty or default, the latest default Corefile will be applied 165 if err := apiclient.CreateOrUpdateConfigMap(client, coreDNSConfigMap); err != nil { 166 return err 167 } 168 } else if corefileMigrationRequired { 169 // If migration is required, try and migrate the Corefile 170 if err := migrateCoreDNSCorefile(client, coreDNSConfigMap, corefile, currentInstalledCoreDNSVersion); err != nil { 171 // Errors in Corefile Migration is verified during preflight checks. This part will be executed when a user has chosen 172 // to ignore preflight check errors. 173 canMigrateCorefile = false 174 klog.Warningf("the CoreDNS Configuration was not migrated: %v. The existing CoreDNS Corefile configuration has been retained.", err) 175 if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNSConfigMap); err != nil { 176 return err 177 } 178 } 179 } else { 180 // If the Corefile is modified and doesn't require any migration, it'll be retained for the benefit of the user 181 if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNSConfigMap); err != nil { 182 return err 183 } 184 } 185 186 coreDNSClusterRoles := &rbac.ClusterRole{} 187 if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(CoreDNSClusterRole), coreDNSClusterRoles); err != nil { 188 return errors.Wrapf(err, "%s ClusterRole", unableToDecodeCoreDNS) 189 } 190 191 // Create the Clusterroles for CoreDNS or update it in case it already exists 192 if err := apiclient.CreateOrUpdateClusterRole(client, coreDNSClusterRoles); err != nil { 193 return err 194 } 195 196 coreDNSClusterRolesBinding := &rbac.ClusterRoleBinding{} 197 if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(CoreDNSClusterRoleBinding), coreDNSClusterRolesBinding); err != nil { 198 return errors.Wrapf(err, "%s ClusterRoleBinding", unableToDecodeCoreDNS) 199 } 200 201 // Create the Clusterrolebindings for CoreDNS or update it in case it already exists 202 if err := apiclient.CreateOrUpdateClusterRoleBinding(client, coreDNSClusterRolesBinding); err != nil { 203 return err 204 } 205 206 coreDNSServiceAccount := &v1.ServiceAccount{} 207 if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(CoreDNSServiceAccount), coreDNSServiceAccount); err != nil { 208 return errors.Wrapf(err, "%s ServiceAccount", unableToDecodeCoreDNS) 209 } 210 211 // Create the ConfigMap for CoreDNS or update it in case it already exists 212 if err := apiclient.CreateOrUpdateServiceAccount(client, coreDNSServiceAccount); err != nil { 213 return err 214 } 215 216 coreDNSDeployment := &apps.Deployment{} 217 if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), deploymentBytes, coreDNSDeployment); err != nil { 218 return errors.Wrapf(err, "%s Deployment", unableToDecodeCoreDNS) 219 } 220 221 // Create the deployment for CoreDNS or retain it in case the CoreDNS migration has failed during upgrade 222 if !canMigrateCorefile { 223 if err := apiclient.CreateOrRetainDeployment(client, coreDNSDeployment, kubeadmconstants.CoreDNSDeploymentName); err != nil { 224 return err 225 } 226 } else { 227 // Create the Deployment for CoreDNS or update it in case it already exists 228 if err := apiclient.CreateOrUpdateDeployment(client, coreDNSDeployment); err != nil { 229 return err 230 } 231 } 232 233 coreDNSService := &v1.Service{} 234 return createDNSService(coreDNSService, serviceBytes, client) 235} 236 237func createDNSService(dnsService *v1.Service, serviceBytes []byte, client clientset.Interface) error { 238 if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), serviceBytes, dnsService); err != nil { 239 return errors.Wrap(err, "unable to decode the DNS service") 240 } 241 242 // Can't use a generic apiclient helper func here as we have to tolerate more than AlreadyExists. 243 if _, err := client.CoreV1().Services(metav1.NamespaceSystem).Create(context.TODO(), dnsService, metav1.CreateOptions{}); err != nil { 244 // Ignore if the Service is invalid with this error message: 245 // Service "kube-dns" is invalid: spec.clusterIP: Invalid value: "10.96.0.10": provided IP is already allocated 246 247 if !apierrors.IsAlreadyExists(err) && !apierrors.IsInvalid(err) { 248 return errors.Wrap(err, "unable to create a new DNS service") 249 } 250 251 if _, err := client.CoreV1().Services(metav1.NamespaceSystem).Update(context.TODO(), dnsService, metav1.UpdateOptions{}); err != nil { 252 return errors.Wrap(err, "unable to create/update the DNS service") 253 } 254 } 255 return nil 256} 257 258// isCoreDNSConfigMapMigrationRequired checks if a migration of the CoreDNS ConfigMap is required. 259func isCoreDNSConfigMapMigrationRequired(corefile, currentInstalledCoreDNSVersion string) (bool, error) { 260 var isMigrationRequired bool 261 262 // Current installed version is expected to be empty for init 263 if currentInstalledCoreDNSVersion == "" { 264 return isMigrationRequired, nil 265 } 266 currentInstalledCoreDNSVersion = strings.TrimLeft(currentInstalledCoreDNSVersion, "v") 267 deprecated, err := migration.Deprecated(currentInstalledCoreDNSVersion, strings.TrimLeft(kubeadmconstants.CoreDNSVersion, "v"), corefile) 268 if err != nil { 269 return isMigrationRequired, errors.Wrap(err, "unable to get list of changes to the configuration.") 270 } 271 272 // Check if there are any plugins/options which needs to be removed or is a new default 273 for _, dep := range deprecated { 274 if dep.Severity == "removed" || dep.Severity == "newdefault" { 275 isMigrationRequired = true 276 } 277 } 278 279 return isMigrationRequired, nil 280} 281 282func migrateCoreDNSCorefile(client clientset.Interface, cm *v1.ConfigMap, corefile, currentInstalledCoreDNSVersion string) error { 283 // Since the current configuration present is not the default version, try and migrate it. 284 updatedCorefile, err := migration.Migrate(currentInstalledCoreDNSVersion, strings.TrimLeft(kubeadmconstants.CoreDNSVersion, "v"), corefile, false) 285 if err != nil { 286 return errors.Wrap(err, "unable to migrate CoreDNS ConfigMap") 287 } 288 289 // Take a copy of the existing Corefile data as `Corefile-backup` and update the ConfigMap 290 if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Update(context.TODO(), &v1.ConfigMap{ 291 ObjectMeta: metav1.ObjectMeta{ 292 Name: kubeadmconstants.CoreDNSConfigMap, 293 Namespace: metav1.NamespaceSystem, 294 }, 295 Data: map[string]string{ 296 "Corefile": updatedCorefile, 297 "Corefile-backup": corefile, 298 }, 299 }, metav1.UpdateOptions{}); err != nil { 300 return errors.Wrap(err, "unable to update the CoreDNS ConfigMap") 301 } 302 303 // Point the CoreDNS deployment to the `Corefile-backup` data. 304 if err := setCorefile(client, "Corefile-backup"); err != nil { 305 return err 306 } 307 308 fmt.Println("[addons] Migrating CoreDNS Corefile") 309 changes, err := migration.Deprecated(currentInstalledCoreDNSVersion, strings.TrimLeft(kubeadmconstants.CoreDNSVersion, "v"), corefile) 310 if err != nil { 311 return errors.Wrap(err, "unable to get list of changes to the configuration.") 312 } 313 // show the migration changes 314 klog.V(2).Infof("the CoreDNS configuration has been migrated and applied: %v.", updatedCorefile) 315 klog.V(2).Infoln("the old migration has been saved in the CoreDNS ConfigMap under the name [Corefile-backup]") 316 klog.V(2).Infoln("The changes in the new CoreDNS Configuration are as follows:") 317 for _, change := range changes { 318 klog.V(2).Infof("%v", change.ToString()) 319 } 320 return nil 321} 322 323// GetCoreDNSInfo gets the current CoreDNS installed and the current Corefile Configuration of CoreDNS. 324func GetCoreDNSInfo(client clientset.Interface) (*v1.ConfigMap, string, string, error) { 325 coreDNSConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.CoreDNSConfigMap, metav1.GetOptions{}) 326 if err != nil && !apierrors.IsNotFound(err) { 327 return nil, "", "", err 328 } 329 if apierrors.IsNotFound(err) { 330 return nil, "", "", nil 331 } 332 corefile, ok := coreDNSConfigMap.Data["Corefile"] 333 if !ok { 334 return nil, "", "", errors.New("unable to find the CoreDNS Corefile data") 335 } 336 337 currentCoreDNSversion, err := DeployedDNSAddon(client) 338 if err != nil { 339 return nil, "", "", err 340 } 341 342 return coreDNSConfigMap, corefile, currentCoreDNSversion, nil 343} 344 345func setCorefile(client clientset.Interface, coreDNSCorefileName string) error { 346 dnsDeployment, err := client.AppsV1().Deployments(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.CoreDNSDeploymentName, metav1.GetOptions{}) 347 if err != nil { 348 return err 349 } 350 patch := fmt.Sprintf(`{"spec":{"template":{"spec":{"volumes":[{"name": "config-volume", "configMap":{"name": "coredns", "items":[{"key": "%s", "path": "Corefile"}]}}]}}}}`, coreDNSCorefileName) 351 352 if _, err := client.AppsV1().Deployments(dnsDeployment.ObjectMeta.Namespace).Patch(context.TODO(), dnsDeployment.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { 353 return errors.Wrap(err, "unable to patch the CoreDNS deployment") 354 } 355 return nil 356} 357