1// Test suite for rclonefs
2
3package vfstest
4
5import (
6	"context"
7	"flag"
8	"fmt"
9	"io"
10	"io/ioutil"
11	"log"
12	"os"
13	"os/exec"
14	"path"
15	"path/filepath"
16	"reflect"
17	"runtime"
18	"strings"
19	"testing"
20	"time"
21
22	_ "github.com/rclone/rclone/backend/all" // import all the backends
23	"github.com/rclone/rclone/cmd/mountlib"
24	"github.com/rclone/rclone/fs"
25	"github.com/rclone/rclone/fs/walk"
26	"github.com/rclone/rclone/fstest"
27	"github.com/rclone/rclone/lib/file"
28	"github.com/rclone/rclone/vfs"
29	"github.com/rclone/rclone/vfs/vfscommon"
30	"github.com/rclone/rclone/vfs/vfsflags"
31	"github.com/stretchr/testify/assert"
32	"github.com/stretchr/testify/require"
33)
34
35const (
36	waitForWritersDelay = 30 * time.Second // time to wait for existing writers
37)
38
39var (
40	mountFn mountlib.MountFn
41)
42
43// RunTests runs all the tests against all the VFS cache modes
44//
45// If useVFS is set then it runs the tests against a VFS rather than amount
46func RunTests(t *testing.T, useVFS bool, fn mountlib.MountFn) {
47	mountFn = fn
48	flag.Parse()
49	tests := []struct {
50		cacheMode vfscommon.CacheMode
51		writeBack time.Duration
52	}{
53		{cacheMode: vfscommon.CacheModeOff},
54		{cacheMode: vfscommon.CacheModeMinimal},
55		{cacheMode: vfscommon.CacheModeWrites},
56		{cacheMode: vfscommon.CacheModeFull},
57		{cacheMode: vfscommon.CacheModeFull, writeBack: 100 * time.Millisecond},
58	}
59	run = newRun(useVFS)
60	for _, test := range tests {
61		run.cacheMode(test.cacheMode, test.writeBack)
62		what := fmt.Sprintf("CacheMode=%v", test.cacheMode)
63		if test.writeBack > 0 {
64			what += fmt.Sprintf(",WriteBack=%v", test.writeBack)
65		}
66		log.Printf("Starting test run with %s", what)
67		ok := t.Run(what, func(t *testing.T) {
68			t.Run("TestTouchAndDelete", TestTouchAndDelete)
69			t.Run("TestRenameOpenHandle", TestRenameOpenHandle)
70			t.Run("TestDirLs", TestDirLs)
71			t.Run("TestDirCreateAndRemoveDir", TestDirCreateAndRemoveDir)
72			t.Run("TestDirCreateAndRemoveFile", TestDirCreateAndRemoveFile)
73			t.Run("TestDirRenameFile", TestDirRenameFile)
74			t.Run("TestDirRenameEmptyDir", TestDirRenameEmptyDir)
75			t.Run("TestDirRenameFullDir", TestDirRenameFullDir)
76			t.Run("TestDirModTime", TestDirModTime)
77			t.Run("TestDirCacheFlush", TestDirCacheFlush)
78			t.Run("TestDirCacheFlushOnDirRename", TestDirCacheFlushOnDirRename)
79			t.Run("TestFileModTime", TestFileModTime)
80			t.Run("TestFileModTimeWithOpenWriters", TestFileModTimeWithOpenWriters)
81			t.Run("TestMount", TestMount)
82			t.Run("TestRoot", TestRoot)
83			t.Run("TestReadByByte", TestReadByByte)
84			t.Run("TestReadChecksum", TestReadChecksum)
85			t.Run("TestReadFileDoubleClose", TestReadFileDoubleClose)
86			t.Run("TestReadSeek", TestReadSeek)
87			t.Run("TestWriteFileNoWrite", TestWriteFileNoWrite)
88			t.Run("TestWriteFileWrite", TestWriteFileWrite)
89			t.Run("TestWriteFileOverwrite", TestWriteFileOverwrite)
90			t.Run("TestWriteFileDoubleClose", TestWriteFileDoubleClose)
91			t.Run("TestWriteFileFsync", TestWriteFileFsync)
92			t.Run("TestWriteFileDup", TestWriteFileDup)
93			t.Run("TestWriteFileAppend", TestWriteFileAppend)
94		})
95		log.Printf("Finished test run with %s (ok=%v)", what, ok)
96		if !ok {
97			break
98		}
99	}
100	run.Finalise()
101}
102
103// Run holds the remotes for a test run
104type Run struct {
105	os           Oser
106	vfs          *vfs.VFS
107	useVFS       bool // set if we are testing a VFS not a mount
108	mountPath    string
109	fremote      fs.Fs
110	fremoteName  string
111	cleanRemote  func()
112	umountResult <-chan error
113	umountFn     mountlib.UnmountFn
114	skip         bool
115}
116
117// run holds the master Run data
118var run *Run
119
120// newRun initialise the remote mount for testing and returns a run
121// object.
122//
123// r.fremote is an empty remote Fs
124//
125// Finalise() will tidy them away when done.
126func newRun(useVFS bool) *Run {
127	r := &Run{
128		useVFS:       useVFS,
129		umountResult: make(chan error, 1),
130	}
131	fstest.Initialise()
132
133	var err error
134	r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote()
135	if err != nil {
136		log.Fatalf("Failed to open remote %q: %v", *fstest.RemoteName, err)
137	}
138
139	err = r.fremote.Mkdir(context.Background(), "")
140	if err != nil {
141		log.Fatalf("Failed to open mkdir %q: %v", *fstest.RemoteName, err)
142	}
143
144	if !r.useVFS {
145		r.mountPath = findMountPath()
146	}
147	// Mount it up
148	r.mount()
149
150	return r
151}
152
153func findMountPath() string {
154	if runtime.GOOS != "windows" {
155		mountPath, err := ioutil.TempDir("", "rclonefs-mount")
156		if err != nil {
157			log.Fatalf("Failed to create mount dir: %v", err)
158		}
159		return mountPath
160	}
161
162	// Find a free drive letter
163	letter := file.FindUnusedDriveLetter()
164	drive := ""
165	if letter == 0 {
166		log.Fatalf("Couldn't find free drive letter for test")
167	} else {
168		drive = string(letter) + ":"
169	}
170	return drive
171}
172
173func (r *Run) mount() {
174	log.Printf("mount %q %q", r.fremote, r.mountPath)
175	var err error
176	r.vfs = vfs.New(r.fremote, &vfsflags.Opt)
177	r.umountResult, r.umountFn, err = mountFn(r.vfs, r.mountPath, &mountlib.Opt)
178	if err != nil {
179		log.Printf("mount FAILED: %v", err)
180		r.skip = true
181	} else {
182		log.Printf("mount OK")
183	}
184	if r.useVFS {
185		r.os = vfsOs{r.vfs}
186	} else {
187		r.os = realOs{}
188	}
189
190}
191
192func (r *Run) umount() {
193	if r.skip {
194		log.Printf("FUSE not found so skipping umount")
195		return
196	}
197	/*
198		log.Printf("Calling fusermount -u %q", r.mountPath)
199		err := exec.Command("fusermount", "-u", r.mountPath).Run()
200		if err != nil {
201			log.Printf("fusermount failed: %v", err)
202		}
203	*/
204	log.Printf("Unmounting %q", r.mountPath)
205	err := r.umountFn()
206	if err != nil {
207		log.Printf("signal to umount failed - retrying: %v", err)
208		time.Sleep(3 * time.Second)
209		err = r.umountFn()
210	}
211	if err != nil {
212		log.Fatalf("signal to umount failed: %v", err)
213	}
214	log.Printf("Waiting for umount")
215	err = <-r.umountResult
216	if err != nil {
217		log.Fatalf("umount failed: %v", err)
218	}
219
220	// Cleanup the VFS cache - umount has called Shutdown
221	err = r.vfs.CleanUp()
222	if err != nil {
223		log.Printf("Failed to cleanup the VFS cache: %v", err)
224	}
225}
226
227// cacheMode flushes the VFS and changes the CacheMode and the writeBack time
228func (r *Run) cacheMode(cacheMode vfscommon.CacheMode, writeBack time.Duration) {
229	if r.skip {
230		log.Printf("FUSE not found so skipping cacheMode")
231		return
232	}
233	// Wait for writers to finish
234	r.vfs.WaitForWriters(waitForWritersDelay)
235	// Empty and remake the remote
236	r.cleanRemote()
237	err := r.fremote.Mkdir(context.Background(), "")
238	if err != nil {
239		log.Fatalf("Failed to open mkdir %q: %v", *fstest.RemoteName, err)
240	}
241	// Empty the cache
242	err = r.vfs.CleanUp()
243	if err != nil {
244		log.Printf("Failed to cleanup the VFS cache: %v", err)
245	}
246	// Reset the cache mode
247	r.vfs.SetCacheMode(cacheMode)
248	r.vfs.Opt.WriteBack = writeBack
249	// Flush the directory cache
250	r.vfs.FlushDirCache()
251
252}
253
254func (r *Run) skipIfNoFUSE(t *testing.T) {
255	if r.skip {
256		t.Skip("FUSE not found so skipping test")
257	}
258}
259
260func (r *Run) skipIfVFS(t *testing.T) {
261	if r.useVFS {
262		t.Skip("Not running under VFS")
263	}
264}
265
266// Finalise cleans the remote and unmounts
267func (r *Run) Finalise() {
268	r.umount()
269	r.cleanRemote()
270	if r.useVFS {
271		// FIXME
272	} else {
273		err := os.RemoveAll(r.mountPath)
274		if err != nil {
275			log.Printf("Failed to clean mountPath %q: %v", r.mountPath, err)
276		}
277	}
278}
279
280// path returns an OS local path for filepath
281func (r *Run) path(filePath string) string {
282	if r.useVFS {
283		return filePath
284	}
285	// return windows drive letter root as E:\
286	if filePath == "" && runtime.GOOS == "windows" {
287		return run.mountPath + `\`
288	}
289	return filepath.Join(run.mountPath, filepath.FromSlash(filePath))
290}
291
292type dirMap map[string]struct{}
293
294// Create a dirMap from a string
295func newDirMap(dirString string) (dm dirMap) {
296	dm = make(dirMap)
297	for _, entry := range strings.Split(dirString, "|") {
298		if entry != "" {
299			dm[entry] = struct{}{}
300		}
301	}
302	return dm
303}
304
305// Returns a dirmap with only the files in
306func (dm dirMap) filesOnly() dirMap {
307	newDm := make(dirMap)
308	for name := range dm {
309		if !strings.HasSuffix(name, "/") {
310			newDm[name] = struct{}{}
311		}
312	}
313	return newDm
314}
315
316// reads the local tree into dir
317func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
318	realPath := r.path(filePath)
319	files, err := r.os.ReadDir(realPath)
320	require.NoError(t, err)
321	for _, fi := range files {
322		name := path.Join(filePath, fi.Name())
323		if fi.IsDir() {
324			dir[name+"/"] = struct{}{}
325			r.readLocal(t, dir, name)
326			assert.Equal(t, run.vfs.Opt.DirPerms&os.ModePerm, fi.Mode().Perm())
327		} else {
328			dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{}
329			assert.Equal(t, run.vfs.Opt.FilePerms&os.ModePerm, fi.Mode().Perm())
330		}
331	}
332}
333
334// reads the remote tree into dir
335func (r *Run) readRemote(t *testing.T, dir dirMap, filepath string) {
336	objs, dirs, err := walk.GetAll(context.Background(), r.fremote, filepath, true, 1)
337	if err == fs.ErrorDirNotFound {
338		return
339	}
340	require.NoError(t, err)
341	for _, obj := range objs {
342		dir[fmt.Sprintf("%s %d", obj.Remote(), obj.Size())] = struct{}{}
343	}
344	for _, d := range dirs {
345		name := d.Remote()
346		dir[name+"/"] = struct{}{}
347		r.readRemote(t, dir, name)
348	}
349}
350
351// checkDir checks the local and remote against the string passed in
352func (r *Run) checkDir(t *testing.T, dirString string) {
353	var retries = *fstest.ListRetries
354	sleep := time.Second / 5
355	var remoteOK, fuseOK bool
356	var dm, localDm, remoteDm dirMap
357	for i := 1; i <= retries; i++ {
358		dm = newDirMap(dirString)
359		localDm = make(dirMap)
360		r.readLocal(t, localDm, "")
361		remoteDm = make(dirMap)
362		r.readRemote(t, remoteDm, "")
363		// Ignore directories for remote compare
364		remoteOK = reflect.DeepEqual(dm.filesOnly(), remoteDm.filesOnly())
365		fuseOK = reflect.DeepEqual(dm, localDm)
366		if remoteOK && fuseOK {
367			return
368		}
369		sleep *= 2
370		t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
371		time.Sleep(sleep)
372	}
373	assert.Equal(t, dm.filesOnly(), remoteDm.filesOnly(), "expected vs remote")
374	assert.Equal(t, dm, localDm, "expected vs fuse mount")
375}
376
377// wait for any files being written to be released by fuse
378func (r *Run) waitForWriters() {
379	run.vfs.WaitForWriters(waitForWritersDelay)
380}
381
382// writeFile writes data to a file named by filename.
383// If the file does not exist, WriteFile creates it with permissions perm;
384// otherwise writeFile truncates it before writing.
385// If there is an error writing then writeFile
386// deletes it an existing file and tries again.
387func writeFile(filename string, data []byte, perm os.FileMode) error {
388	f, err := run.os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
389	if err != nil {
390		err = run.os.Remove(filename)
391		if err != nil {
392			return err
393		}
394		f, err = run.os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, perm)
395		if err != nil {
396			return err
397		}
398	}
399	n, err := f.Write(data)
400	if err == nil && n < len(data) {
401		err = io.ErrShortWrite
402	}
403	if err1 := f.Close(); err == nil {
404		err = err1
405	}
406	return err
407}
408
409func (r *Run) createFile(t *testing.T, filepath string, contents string) {
410	filepath = r.path(filepath)
411	err := writeFile(filepath, []byte(contents), 0600)
412	require.NoError(t, err)
413	r.waitForWriters()
414}
415
416func (r *Run) readFile(t *testing.T, filepath string) string {
417	filepath = r.path(filepath)
418	result, err := run.os.ReadFile(filepath)
419	require.NoError(t, err)
420	return string(result)
421}
422
423func (r *Run) mkdir(t *testing.T, filepath string) {
424	filepath = r.path(filepath)
425	err := run.os.Mkdir(filepath, 0700)
426	require.NoError(t, err)
427}
428
429func (r *Run) rm(t *testing.T, filepath string) {
430	filepath = r.path(filepath)
431	err := run.os.Remove(filepath)
432	require.NoError(t, err)
433
434	// Wait for file to disappear from listing
435	for i := 0; i < 100; i++ {
436		_, err := run.os.Stat(filepath)
437		if os.IsNotExist(err) {
438			return
439		}
440		time.Sleep(100 * time.Millisecond)
441	}
442	assert.Fail(t, "failed to delete file", filepath)
443}
444
445func (r *Run) rmdir(t *testing.T, filepath string) {
446	filepath = r.path(filepath)
447	err := run.os.Remove(filepath)
448	require.NoError(t, err)
449}
450
451// TestMount checks that the Fs is mounted by seeing if the mountpoint
452// is in the mount output
453func TestMount(t *testing.T) {
454	run.skipIfVFS(t)
455	run.skipIfNoFUSE(t)
456	if runtime.GOOS == "windows" {
457		t.Skip("not running on windows")
458	}
459
460	out, err := exec.Command("mount").Output()
461	require.NoError(t, err)
462	assert.Contains(t, string(out), run.mountPath)
463}
464
465// TestRoot checks root directory is present and correct
466func TestRoot(t *testing.T) {
467	run.skipIfVFS(t)
468	run.skipIfNoFUSE(t)
469
470	fi, err := os.Lstat(run.mountPath)
471	require.NoError(t, err)
472	assert.True(t, fi.IsDir())
473	assert.Equal(t, run.vfs.Opt.DirPerms&os.ModePerm, fi.Mode().Perm())
474}
475