1/*
2Copyright 2018 The Kubernetes Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8    http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17package volumepathhandler
18
19import (
20	"fmt"
21	"io/ioutil"
22	"os"
23	"path/filepath"
24
25	"k8s.io/klog/v2"
26	"k8s.io/mount-utils"
27	utilexec "k8s.io/utils/exec"
28
29	"k8s.io/apimachinery/pkg/types"
30)
31
32const (
33	losetupPath       = "losetup"
34	ErrDeviceNotFound = "device not found"
35)
36
37// BlockVolumePathHandler defines a set of operations for handling block volume-related operations
38type BlockVolumePathHandler interface {
39	// MapDevice creates a symbolic link to block device under specified map path
40	MapDevice(devicePath string, mapPath string, linkName string, bindMount bool) error
41	// UnmapDevice removes a symbolic link to block device under specified map path
42	UnmapDevice(mapPath string, linkName string, bindMount bool) error
43	// RemovePath removes a file or directory on specified map path
44	RemoveMapPath(mapPath string) error
45	// IsSymlinkExist retruns true if specified symbolic link exists
46	IsSymlinkExist(mapPath string) (bool, error)
47	// IsDeviceBindMountExist retruns true if specified bind mount exists
48	IsDeviceBindMountExist(mapPath string) (bool, error)
49	// GetDeviceBindMountRefs searches bind mounts under global map path
50	GetDeviceBindMountRefs(devPath string, mapPath string) ([]string, error)
51	// FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath
52	// corresponding to map path symlink, and then return global map path with pod uuid.
53	FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error)
54	// AttachFileDevice takes a path to a regular file and makes it available as an
55	// attached block device.
56	AttachFileDevice(path string) (string, error)
57	// DetachFileDevice takes a path to the attached block device and
58	// detach it from block device.
59	DetachFileDevice(path string) error
60	// GetLoopDevice returns the full path to the loop device associated with the given path.
61	GetLoopDevice(path string) (string, error)
62}
63
64// NewBlockVolumePathHandler returns a new instance of BlockVolumeHandler.
65func NewBlockVolumePathHandler() BlockVolumePathHandler {
66	var volumePathHandler VolumePathHandler
67	return volumePathHandler
68}
69
70// VolumePathHandler is path related operation handlers for block volume
71type VolumePathHandler struct {
72}
73
74// MapDevice creates a symbolic link to block device under specified map path
75func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName string, bindMount bool) error {
76	// Example of global map path:
77	//   globalMapPath/linkName: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{podUid}
78	//   linkName: {podUid}
79	//
80	// Example of pod device map path:
81	//   podDeviceMapPath/linkName: pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
82	//   linkName: {volumeName}
83	if len(devicePath) == 0 {
84		return fmt.Errorf("failed to map device to map path. devicePath is empty")
85	}
86	if len(mapPath) == 0 {
87		return fmt.Errorf("failed to map device to map path. mapPath is empty")
88	}
89	if !filepath.IsAbs(mapPath) {
90		return fmt.Errorf("the map path should be absolute: map path: %s", mapPath)
91	}
92	klog.V(5).Infof("MapDevice: devicePath %s", devicePath)
93	klog.V(5).Infof("MapDevice: mapPath %s", mapPath)
94	klog.V(5).Infof("MapDevice: linkName %s", linkName)
95
96	// Check and create mapPath
97	_, err := os.Stat(mapPath)
98	if err != nil && !os.IsNotExist(err) {
99		return fmt.Errorf("cannot validate map path: %s: %v", mapPath, err)
100	}
101	if err = os.MkdirAll(mapPath, 0750); err != nil {
102		return fmt.Errorf("failed to mkdir %s: %v", mapPath, err)
103	}
104
105	if bindMount {
106		return mapBindMountDevice(v, devicePath, mapPath, linkName)
107	}
108	return mapSymlinkDevice(v, devicePath, mapPath, linkName)
109}
110
111func mapBindMountDevice(v VolumePathHandler, devicePath string, mapPath string, linkName string) error {
112	// Check bind mount exists
113	linkPath := filepath.Join(mapPath, string(linkName))
114
115	file, err := os.Stat(linkPath)
116	if err != nil {
117		if !os.IsNotExist(err) {
118			return fmt.Errorf("failed to stat file %s: %v", linkPath, err)
119		}
120
121		// Create file
122		newFile, err := os.OpenFile(linkPath, os.O_CREATE|os.O_RDWR, 0750)
123		if err != nil {
124			return fmt.Errorf("failed to open file %s: %v", linkPath, err)
125		}
126		if err := newFile.Close(); err != nil {
127			return fmt.Errorf("failed to close file %s: %v", linkPath, err)
128		}
129	} else {
130		// Check if device file
131		// TODO: Need to check if this device file is actually the expected bind mount
132		if file.Mode()&os.ModeDevice == os.ModeDevice {
133			klog.Warningf("Warning: Map skipped because bind mount already exist on the path: %v", linkPath)
134			return nil
135		}
136
137		klog.Warningf("Warning: file %s is already exist but not mounted, skip creating file", linkPath)
138	}
139
140	// Bind mount file
141	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()}
142	if err := mounter.MountSensitiveWithoutSystemd(devicePath, linkPath, "" /* fsType */, []string{"bind"}, nil); err != nil {
143		return fmt.Errorf("failed to bind mount devicePath: %s to linkPath %s: %v", devicePath, linkPath, err)
144	}
145
146	return nil
147}
148
149func mapSymlinkDevice(v VolumePathHandler, devicePath string, mapPath string, linkName string) error {
150	// Remove old symbolic link(or file) then create new one.
151	// This should be done because current symbolic link is
152	// stale across node reboot.
153	linkPath := filepath.Join(mapPath, string(linkName))
154	if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
155		return fmt.Errorf("failed to remove file %s: %v", linkPath, err)
156	}
157	return os.Symlink(devicePath, linkPath)
158}
159
160// UnmapDevice removes a symbolic link associated to block device under specified map path
161func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string, bindMount bool) error {
162	if len(mapPath) == 0 {
163		return fmt.Errorf("failed to unmap device from map path. mapPath is empty")
164	}
165	klog.V(5).Infof("UnmapDevice: mapPath %s", mapPath)
166	klog.V(5).Infof("UnmapDevice: linkName %s", linkName)
167
168	if bindMount {
169		return unmapBindMountDevice(v, mapPath, linkName)
170	}
171	return unmapSymlinkDevice(v, mapPath, linkName)
172}
173
174func unmapBindMountDevice(v VolumePathHandler, mapPath string, linkName string) error {
175	// Check bind mount exists
176	linkPath := filepath.Join(mapPath, string(linkName))
177	if isMountExist, checkErr := v.IsDeviceBindMountExist(linkPath); checkErr != nil {
178		return checkErr
179	} else if !isMountExist {
180		klog.Warningf("Warning: Unmap skipped because bind mount does not exist on the path: %v", linkPath)
181
182		// Check if linkPath still exists
183		if _, err := os.Stat(linkPath); err != nil {
184			if !os.IsNotExist(err) {
185				return fmt.Errorf("failed to check if path %s exists: %v", linkPath, err)
186			}
187			// linkPath has already been removed
188			return nil
189		}
190		// Remove file
191		if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
192			return fmt.Errorf("failed to remove file %s: %v", linkPath, err)
193		}
194		return nil
195	}
196
197	// Unmount file
198	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()}
199	if err := mounter.Unmount(linkPath); err != nil {
200		return fmt.Errorf("failed to unmount linkPath %s: %v", linkPath, err)
201	}
202
203	// Remove file
204	if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
205		return fmt.Errorf("failed to remove file %s: %v", linkPath, err)
206	}
207
208	return nil
209}
210
211func unmapSymlinkDevice(v VolumePathHandler, mapPath string, linkName string) error {
212	// Check symbolic link exists
213	linkPath := filepath.Join(mapPath, string(linkName))
214	if islinkExist, checkErr := v.IsSymlinkExist(linkPath); checkErr != nil {
215		return checkErr
216	} else if !islinkExist {
217		klog.Warningf("Warning: Unmap skipped because symlink does not exist on the path: %v", linkPath)
218		return nil
219	}
220	return os.Remove(linkPath)
221}
222
223// RemoveMapPath removes a file or directory on specified map path
224func (v VolumePathHandler) RemoveMapPath(mapPath string) error {
225	if len(mapPath) == 0 {
226		return fmt.Errorf("failed to remove map path. mapPath is empty")
227	}
228	klog.V(5).Infof("RemoveMapPath: mapPath %s", mapPath)
229	err := os.RemoveAll(mapPath)
230	if err != nil && !os.IsNotExist(err) {
231		return fmt.Errorf("failed to remove directory %s: %v", mapPath, err)
232	}
233	return nil
234}
235
236// IsSymlinkExist returns true if specified file exists and the type is symbolik link.
237// If file doesn't exist, or file exists but not symbolic link, return false with no error.
238// On other cases, return false with error from Lstat().
239func (v VolumePathHandler) IsSymlinkExist(mapPath string) (bool, error) {
240	fi, err := os.Lstat(mapPath)
241	if err != nil {
242		// If file doesn't exist, return false and no error
243		if os.IsNotExist(err) {
244			return false, nil
245		}
246		// Return error from Lstat()
247		return false, fmt.Errorf("failed to Lstat file %s: %v", mapPath, err)
248	}
249	// If file exits and it's symbolic link, return true and no error
250	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
251		return true, nil
252	}
253	// If file exits but it's not symbolic link, return false and no error
254	return false, nil
255}
256
257// IsDeviceBindMountExist returns true if specified file exists and the type is device.
258// If file doesn't exist, or file exists but not device, return false with no error.
259// On other cases, return false with error from Lstat().
260func (v VolumePathHandler) IsDeviceBindMountExist(mapPath string) (bool, error) {
261	fi, err := os.Lstat(mapPath)
262	if err != nil {
263		// If file doesn't exist, return false and no error
264		if os.IsNotExist(err) {
265			return false, nil
266		}
267
268		// Return error from Lstat()
269		return false, fmt.Errorf("failed to Lstat file %s: %v", mapPath, err)
270	}
271	// If file exits and it's device, return true and no error
272	if fi.Mode()&os.ModeDevice == os.ModeDevice {
273		return true, nil
274	}
275	// If file exits but it's not device, return false and no error
276	return false, nil
277}
278
279// GetDeviceBindMountRefs searches bind mounts under global map path
280func (v VolumePathHandler) GetDeviceBindMountRefs(devPath string, mapPath string) ([]string, error) {
281	var refs []string
282	files, err := ioutil.ReadDir(mapPath)
283	if err != nil {
284		return nil, err
285	}
286	for _, file := range files {
287		if file.Mode()&os.ModeDevice != os.ModeDevice {
288			continue
289		}
290		filename := file.Name()
291		// TODO: Might need to check if the file is actually linked to devPath
292		refs = append(refs, filepath.Join(mapPath, filename))
293	}
294	klog.V(5).Infof("GetDeviceBindMountRefs: refs %v", refs)
295	return refs, nil
296}
297