1/* 2Copyright 2016 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package services 18 19import ( 20 "flag" 21 "fmt" 22 "io/ioutil" 23 "os" 24 "os/exec" 25 "path/filepath" 26 "strings" 27 "time" 28 29 "github.com/spf13/pflag" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 utilfeature "k8s.io/apiserver/pkg/util/feature" 32 cliflag "k8s.io/component-base/cli/flag" 33 "k8s.io/klog/v2" 34 kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" 35 36 "k8s.io/kubernetes/cmd/kubelet/app/options" 37 "k8s.io/kubernetes/pkg/features" 38 kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" 39 kubeletconfigcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec" 40 "k8s.io/kubernetes/test/e2e/framework" 41 "k8s.io/kubernetes/test/e2e_node/builder" 42 "k8s.io/kubernetes/test/e2e_node/remote" 43) 44 45// TODO(random-liu): Replace this with standard kubelet launcher. 46 47// args is the type used to accumulate args from the flags with the same name. 48type args []string 49 50// String function of flag.Value 51func (a *args) String() string { 52 return fmt.Sprint(*a) 53} 54 55// Set function of flag.Value 56func (a *args) Set(value string) error { 57 // Note that we assume all white space in flag string is separating fields 58 na := strings.Fields(value) 59 *a = append(*a, na...) 60 return nil 61} 62 63// kubeletArgs is the override kubelet args specified by the test runner. 64var kubeletArgs args 65var genKubeletConfigFile bool 66 67func init() { 68 flag.Var(&kubeletArgs, "kubelet-flags", "Kubelet flags passed to kubelet, this will override default kubelet flags in the test. Flags specified in multiple kubelet-flags will be concatenate.") 69 flag.BoolVar(&genKubeletConfigFile, "generate-kubelet-config-file", true, "The test runner will generate a Kubelet config file containing test defaults instead of passing default flags to the Kubelet.") 70} 71 72// RunKubelet starts kubelet and waits for termination signal. Once receives the 73// termination signal, it will stop the kubelet gracefully. 74func RunKubelet() { 75 var err error 76 // Enable monitorParent to make sure kubelet will receive termination signal 77 // when test process exits. 78 e := NewE2EServices(true /* monitorParent */) 79 defer e.Stop() 80 e.kubelet, err = e.startKubelet() 81 if err != nil { 82 klog.Fatalf("Failed to start kubelet: %v", err) 83 } 84 // Wait until receiving a termination signal. 85 waitForTerminationSignal() 86} 87 88const ( 89 // Ports of different e2e services. 90 kubeletReadOnlyPort = "10255" 91 // KubeletRootDirectory specifies the directory where the kubelet runtime information is stored. 92 KubeletRootDirectory = "/var/lib/kubelet" 93 // Health check url of kubelet 94 kubeletHealthCheckURL = "http://127.0.0.1:" + kubeletReadOnlyPort + "/healthz" 95) 96 97// startKubelet starts the Kubelet in a separate process or returns an error 98// if the Kubelet fails to start. 99func (e *E2EServices) startKubelet() (*server, error) { 100 klog.Info("Starting kubelet") 101 102 // set feature gates so we can check which features are enabled and pass the appropriate flags 103 if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(framework.TestContext.FeatureGates); err != nil { 104 return nil, err 105 } 106 107 // Build kubeconfig 108 kubeconfigPath, err := createKubeconfigCWD() 109 if err != nil { 110 return nil, err 111 } 112 113 // KubeletConfiguration file path 114 kubeletConfigPath, err := kubeletConfigCWDPath() 115 if err != nil { 116 return nil, err 117 } 118 119 // Create pod directory 120 podPath, err := createPodDirectory() 121 if err != nil { 122 return nil, err 123 } 124 e.rmDirs = append(e.rmDirs, podPath) 125 err = createRootDirectory(KubeletRootDirectory) 126 if err != nil { 127 return nil, err 128 } 129 130 // PLEASE NOTE: If you set new KubeletConfiguration values or stop setting values here, 131 // you must also update the flag names in kubeletConfigFlags! 132 kubeletConfigFlags := []string{} 133 134 // set up the default kubeletconfiguration 135 kc, err := options.NewKubeletConfiguration() 136 if err != nil { 137 return nil, err 138 } 139 140 kc.CgroupRoot = "/" 141 kubeletConfigFlags = append(kubeletConfigFlags, "cgroup-root") 142 143 kc.VolumeStatsAggPeriod = metav1.Duration{Duration: 10 * time.Second} // Aggregate volumes frequently so tests don't need to wait as long 144 kubeletConfigFlags = append(kubeletConfigFlags, "volume-stats-agg-period") 145 146 kc.SerializeImagePulls = false 147 kubeletConfigFlags = append(kubeletConfigFlags, "serialize-image-pulls") 148 149 kc.StaticPodPath = podPath 150 kubeletConfigFlags = append(kubeletConfigFlags, "pod-manifest-path") 151 152 kc.FileCheckFrequency = metav1.Duration{Duration: 10 * time.Second} // Check file frequently so tests won't wait too long 153 kubeletConfigFlags = append(kubeletConfigFlags, "file-check-frequency") 154 155 // Assign a fixed CIDR to the node because there is no node controller. 156 // Note: this MUST be in sync with the IP in 157 // - cluster/gce/config-test.sh and 158 // - test/e2e_node/conformance/run_test.sh. 159 kc.PodCIDR = "10.100.0.0/24" 160 kubeletConfigFlags = append(kubeletConfigFlags, "pod-cidr") 161 162 kc.EvictionPressureTransitionPeriod = metav1.Duration{Duration: 30 * time.Second} 163 kubeletConfigFlags = append(kubeletConfigFlags, "eviction-pressure-transition-period") 164 165 kc.EvictionHard = map[string]string{ 166 "memory.available": "250Mi", 167 "nodefs.available": "10%", 168 "nodefs.inodesFree": "5%", 169 } 170 kubeletConfigFlags = append(kubeletConfigFlags, "eviction-hard") 171 172 kc.EvictionMinimumReclaim = map[string]string{ 173 "nodefs.available": "5%", 174 "nodefs.inodesFree": "5%", 175 } 176 kubeletConfigFlags = append(kubeletConfigFlags, "eviction-minimum-reclaim") 177 178 var killCommand, restartCommand *exec.Cmd 179 var isSystemd bool 180 // Apply default kubelet flags. 181 cmdArgs := []string{} 182 if systemdRun, err := exec.LookPath("systemd-run"); err == nil { 183 // On systemd services, detection of a service / unit works reliably while 184 // detection of a process started from an ssh session does not work. 185 // Since kubelet will typically be run as a service it also makes more 186 // sense to test it that way 187 isSystemd = true 188 // We can ignore errors, to have GetTimestampFromWorkspaceDir() fallback 189 // to the current time. 190 cwd, _ := os.Getwd() 191 // Use the timestamp from the current directory to name the systemd unit. 192 unitTimestamp := remote.GetTimestampFromWorkspaceDir(cwd) 193 unitName := fmt.Sprintf("kubelet-%s.service", unitTimestamp) 194 cmdArgs = append(cmdArgs, 195 systemdRun, 196 "-p", "Delegate=true", 197 "-p", "StandardError=file:"+framework.TestContext.ReportDir+"/kubelet.log", 198 "--unit="+unitName, 199 "--slice=runtime.slice", 200 "--remain-after-exit", 201 builder.GetKubeletServerBin()) 202 203 killCommand = exec.Command("systemctl", "kill", unitName) 204 restartCommand = exec.Command("systemctl", "restart", unitName) 205 206 kc.KubeletCgroups = "/kubelet.slice" 207 kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups") 208 } else { 209 cmdArgs = append(cmdArgs, builder.GetKubeletServerBin()) 210 // TODO(random-liu): Get rid of this docker specific thing. 211 cmdArgs = append(cmdArgs, "--runtime-cgroups=/docker-daemon") 212 213 kc.KubeletCgroups = "/kubelet" 214 kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups") 215 216 kc.SystemCgroups = "/system" 217 kubeletConfigFlags = append(kubeletConfigFlags, "system-cgroups") 218 } 219 cmdArgs = append(cmdArgs, 220 "--kubeconfig", kubeconfigPath, 221 "--root-dir", KubeletRootDirectory, 222 "--v", LogVerbosityLevel, "--logtostderr", 223 ) 224 225 // Apply test framework feature gates by default. This could also be overridden 226 // by kubelet-flags. 227 if len(framework.TestContext.FeatureGates) > 0 { 228 cmdArgs = append(cmdArgs, "--feature-gates", cliflag.NewMapStringBool(&framework.TestContext.FeatureGates).String()) 229 kc.FeatureGates = framework.TestContext.FeatureGates 230 } 231 232 if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) { 233 // Enable dynamic config if the feature gate is enabled 234 dynamicConfigDir, err := getDynamicConfigDir() 235 if err != nil { 236 return nil, err 237 } 238 cmdArgs = append(cmdArgs, "--dynamic-config-dir", dynamicConfigDir) 239 } 240 241 // Enable kubenet by default. 242 cniBinDir, err := getCNIBinDirectory() 243 if err != nil { 244 return nil, err 245 } 246 247 cniConfDir, err := getCNIConfDirectory() 248 if err != nil { 249 return nil, err 250 } 251 252 cniCacheDir, err := getCNICacheDirectory() 253 if err != nil { 254 return nil, err 255 } 256 257 cmdArgs = append(cmdArgs, 258 "--network-plugin=kubenet", 259 "--cni-bin-dir", cniBinDir, 260 "--cni-conf-dir", cniConfDir, 261 "--cni-cache-dir", cniCacheDir) 262 263 // Keep hostname override for convenience. 264 if framework.TestContext.NodeName != "" { // If node name is specified, set hostname override. 265 cmdArgs = append(cmdArgs, "--hostname-override", framework.TestContext.NodeName) 266 } 267 268 if framework.TestContext.ContainerRuntime != "" { 269 cmdArgs = append(cmdArgs, "--container-runtime", framework.TestContext.ContainerRuntime) 270 } 271 272 if framework.TestContext.ContainerRuntimeEndpoint != "" { 273 cmdArgs = append(cmdArgs, "--container-runtime-endpoint", framework.TestContext.ContainerRuntimeEndpoint) 274 } 275 276 if framework.TestContext.ImageServiceEndpoint != "" { 277 cmdArgs = append(cmdArgs, "--image-service-endpoint", framework.TestContext.ImageServiceEndpoint) 278 } 279 280 // Write config file or flags, depending on whether --generate-kubelet-config-file was provided 281 if genKubeletConfigFile { 282 if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil { 283 return nil, err 284 } 285 // add the flag to load config from a file 286 cmdArgs = append(cmdArgs, "--config", kubeletConfigPath) 287 } else { 288 // generate command line flags from the default config, since --generate-kubelet-config-file was not provided 289 addKubeletConfigFlags(&cmdArgs, kc, kubeletConfigFlags) 290 } 291 292 // Override the default kubelet flags. 293 cmdArgs = append(cmdArgs, kubeletArgs...) 294 295 // Adjust the args if we are running kubelet with systemd. 296 if isSystemd { 297 adjustArgsForSystemd(cmdArgs) 298 } 299 300 cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) 301 restartOnExit := framework.TestContext.RestartKubelet 302 server := newServer( 303 "kubelet", 304 cmd, 305 killCommand, 306 restartCommand, 307 []string{kubeletHealthCheckURL}, 308 "kubelet.log", 309 e.monitorParent, 310 restartOnExit) 311 return server, server.start() 312} 313 314// addKubeletConfigFlags adds the flags we care about from the provided kubelet configuration object 315func addKubeletConfigFlags(cmdArgs *[]string, kc *kubeletconfig.KubeletConfiguration, flags []string) { 316 fs := pflag.NewFlagSet("kubelet", pflag.ExitOnError) 317 options.AddKubeletConfigFlags(fs, kc) 318 for _, name := range flags { 319 *cmdArgs = append(*cmdArgs, fmt.Sprintf("--%s=%s", name, fs.Lookup(name).Value.String())) 320 } 321} 322 323// writeKubeletConfigFile writes the kubelet config file based on the args and returns the filename 324func writeKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error { 325 data, err := kubeletconfigcodec.EncodeKubeletConfig(internal, kubeletconfigv1beta1.SchemeGroupVersion) 326 if err != nil { 327 return err 328 } 329 // create the directory, if it does not exist 330 dir := filepath.Dir(path) 331 if err := os.MkdirAll(dir, 0755); err != nil { 332 return err 333 } 334 // write the file 335 if err := ioutil.WriteFile(path, data, 0755); err != nil { 336 return err 337 } 338 return nil 339} 340 341// createPodDirectory creates pod directory. 342func createPodDirectory() (string, error) { 343 cwd, err := os.Getwd() 344 if err != nil { 345 return "", fmt.Errorf("failed to get current working directory: %v", err) 346 } 347 path, err := ioutil.TempDir(cwd, "static-pods") 348 if err != nil { 349 return "", fmt.Errorf("failed to create static pod directory: %v", err) 350 } 351 return path, nil 352} 353 354// createKubeconfig creates a kubeconfig file at the fully qualified `path`. The parent dirs must exist. 355func createKubeconfig(path string) error { 356 kubeconfig := []byte(fmt.Sprintf(`apiVersion: v1 357kind: Config 358users: 359- name: kubelet 360 user: 361 token: %s 362clusters: 363- cluster: 364 server: %s 365 insecure-skip-tls-verify: true 366 name: local 367contexts: 368- context: 369 cluster: local 370 user: kubelet 371 name: local-context 372current-context: local-context`, framework.TestContext.BearerToken, getAPIServerClientURL())) 373 374 if err := ioutil.WriteFile(path, kubeconfig, 0666); err != nil { 375 return err 376 } 377 return nil 378} 379 380func createRootDirectory(path string) error { 381 if _, err := os.Stat(path); err != nil { 382 if os.IsNotExist(err) { 383 return os.MkdirAll(path, os.FileMode(0755)) 384 } 385 return err 386 } 387 return nil 388} 389 390func kubeconfigCWDPath() (string, error) { 391 cwd, err := os.Getwd() 392 if err != nil { 393 return "", fmt.Errorf("failed to get current working directory: %v", err) 394 } 395 return filepath.Join(cwd, "kubeconfig"), nil 396} 397 398func kubeletConfigCWDPath() (string, error) { 399 cwd, err := os.Getwd() 400 if err != nil { 401 return "", fmt.Errorf("failed to get current working directory: %v", err) 402 } 403 // DO NOT name this file "kubelet" - you will overwrite the kubelet binary and be very confused :) 404 return filepath.Join(cwd, "kubelet-config"), nil 405} 406 407// like createKubeconfig, but creates kubeconfig at current-working-directory/kubeconfig 408// returns a fully-qualified path to the kubeconfig file 409func createKubeconfigCWD() (string, error) { 410 kubeconfigPath, err := kubeconfigCWDPath() 411 if err != nil { 412 return "", err 413 } 414 415 if err = createKubeconfig(kubeconfigPath); err != nil { 416 return "", err 417 } 418 return kubeconfigPath, nil 419} 420 421// getCNIBinDirectory returns CNI directory. 422func getCNIBinDirectory() (string, error) { 423 cwd, err := os.Getwd() 424 if err != nil { 425 return "", err 426 } 427 return filepath.Join(cwd, "cni", "bin"), nil 428} 429 430// getCNIConfDirectory returns CNI Configuration directory. 431func getCNIConfDirectory() (string, error) { 432 cwd, err := os.Getwd() 433 if err != nil { 434 return "", err 435 } 436 return filepath.Join(cwd, "cni", "net.d"), nil 437} 438 439// getCNICacheDirectory returns CNI Cache directory. 440func getCNICacheDirectory() (string, error) { 441 cwd, err := os.Getwd() 442 if err != nil { 443 return "", err 444 } 445 return filepath.Join(cwd, "cni", "cache"), nil 446} 447 448// getDynamicConfigDir returns the directory for dynamic Kubelet configuration 449func getDynamicConfigDir() (string, error) { 450 cwd, err := os.Getwd() 451 if err != nil { 452 return "", err 453 } 454 return filepath.Join(cwd, "dynamic-kubelet-config"), nil 455} 456 457// adjustArgsForSystemd escape special characters in kubelet arguments for systemd. Systemd 458// may try to do auto expansion without escaping. 459func adjustArgsForSystemd(args []string) { 460 for i := range args { 461 args[i] = strings.Replace(args[i], "%", "%%", -1) 462 args[i] = strings.Replace(args[i], "$", "$$", -1) 463 } 464} 465