1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing,
12 // software distributed under the License is distributed on an
13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 // KIND, either express or implied. See the License for the
15 // specific language governing permissions and limitations
16 // under the License.
17
18 #pragma once
19
20 #include <chrono>
21 #include <memory>
22 #include <string>
23 #include <vector>
24
25 #include "arrow/filesystem/filesystem.h"
26 #include "arrow/filesystem/mockfs.h"
27 #include "arrow/testing/visibility.h"
28 #include "arrow/util/counting_semaphore.h"
29
30 namespace arrow {
31 namespace fs {
32
33 static constexpr double kTimeSlack = 2.0; // In seconds
34
File(std::string path)35 static inline FileInfo File(std::string path) {
36 return FileInfo(std::move(path), FileType::File);
37 }
38
Dir(std::string path)39 static inline FileInfo Dir(std::string path) {
40 return FileInfo(std::move(path), FileType::Directory);
41 }
42
43 // A subclass of MockFileSystem that blocks operations until an unlock method is
44 // called.
45 //
46 // This is intended for testing fine-grained ordering of filesystem operations.
47 //
48 // N.B. Only OpenOutputStream supports gating at the moment but this is simply because
49 // it is all that has been needed so far. Feel free to add support for more methods
50 // as required.
51 class ARROW_TESTING_EXPORT GatedMockFilesystem : public internal::MockFileSystem {
52 public:
53 GatedMockFilesystem(TimePoint current_time,
54 const io::IOContext& = io::default_io_context());
55 ~GatedMockFilesystem() override;
56
57 Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
58 const std::string& path,
59 const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
60
61 // Wait until at least num_waiters are waiting on OpenOutputStream
62 Status WaitForOpenOutputStream(uint32_t num_waiters);
63 // Unlock `num_waiters` individual calls to OpenOutputStream
64 Status UnlockOpenOutputStream(uint32_t num_waiters);
65
66 private:
67 util::CountingSemaphore open_output_sem_;
68 };
69
70 ARROW_TESTING_EXPORT
71 void CreateFile(FileSystem* fs, const std::string& path, const std::string& data);
72
73 // Sort a vector of FileInfo by lexicographic path order
74 ARROW_TESTING_EXPORT
75 void SortInfos(FileInfoVector* infos);
76
77 ARROW_TESTING_EXPORT
78 void CollectFileInfoGenerator(FileInfoGenerator gen, FileInfoVector* out_infos);
79
80 ARROW_TESTING_EXPORT
81 void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type);
82
83 ARROW_TESTING_EXPORT
84 void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type,
85 TimePoint mtime);
86
87 ARROW_TESTING_EXPORT
88 void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type,
89 TimePoint mtime, int64_t size);
90
91 ARROW_TESTING_EXPORT
92 void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type,
93 int64_t size);
94
95 ARROW_TESTING_EXPORT
96 void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type);
97
98 ARROW_TESTING_EXPORT
99 void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type,
100 TimePoint mtime);
101
102 ARROW_TESTING_EXPORT
103 void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type,
104 TimePoint mtime, int64_t size);
105
106 ARROW_TESTING_EXPORT
107 void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type, int64_t size);
108
109 ARROW_TESTING_EXPORT
110 void AssertFileContents(FileSystem* fs, const std::string& path,
111 const std::string& expected_data);
112
113 template <typename Duration>
AssertDurationBetween(Duration d,double min_secs,double max_secs)114 void AssertDurationBetween(Duration d, double min_secs, double max_secs) {
115 auto seconds = std::chrono::duration_cast<std::chrono::duration<double>>(d);
116 ASSERT_GE(seconds.count(), min_secs);
117 ASSERT_LE(seconds.count(), max_secs);
118 }
119
120 // Generic tests for FileSystem implementations.
121 // To use this class, subclass both from it and ::testing::Test,
122 // implement GetEmptyFileSystem(), and use GENERIC_FS_TEST_FUNCTIONS()
123 // to define the various tests.
124 class ARROW_TESTING_EXPORT GenericFileSystemTest {
125 public:
126 virtual ~GenericFileSystemTest();
127
128 void TestEmpty();
129 void TestNormalizePath();
130 void TestCreateDir();
131 void TestDeleteDir();
132 void TestDeleteDirContents();
133 void TestDeleteRootDirContents();
134 void TestDeleteFile();
135 void TestDeleteFiles();
136 void TestMoveFile();
137 void TestMoveDir();
138 void TestCopyFile();
139 void TestGetFileInfo();
140 void TestGetFileInfoVector();
141 void TestGetFileInfoSelector();
142 void TestGetFileInfoSelectorWithRecursion();
143 void TestGetFileInfoAsync();
144 void TestGetFileInfoGenerator();
145 void TestOpenOutputStream();
146 void TestOpenAppendStream();
147 void TestOpenInputStream();
148 void TestOpenInputStreamWithFileInfo();
149 void TestOpenInputStreamAsync();
150 void TestOpenInputFile();
151 void TestOpenInputFileWithFileInfo();
152 void TestOpenInputFileAsync();
153 void TestSpecialChars();
154
155 protected:
156 // This function should return the filesystem under test.
157 virtual std::shared_ptr<FileSystem> GetEmptyFileSystem() = 0;
158
159 // Override the following functions to specify deviations from expected
160 // filesystem semantics.
161 // - Whether the filesystem may "implicitly" create intermediate directories
have_implicit_directories()162 virtual bool have_implicit_directories() const { return false; }
163 // - Whether the filesystem may allow writing a file "over" a directory
allow_write_file_over_dir()164 virtual bool allow_write_file_over_dir() const { return false; }
165 // - Whether the filesystem allows moving a directory
allow_move_dir()166 virtual bool allow_move_dir() const { return true; }
167 // - Whether the filesystem allows moving a directory "over" a non-empty destination
allow_move_dir_over_non_empty_dir()168 virtual bool allow_move_dir_over_non_empty_dir() const { return false; }
169 // - Whether the filesystem allows appending to a file
allow_append_to_file()170 virtual bool allow_append_to_file() const { return true; }
171 // - Whether the filesystem allows appending to a new (not existent yet) file
allow_append_to_new_file()172 virtual bool allow_append_to_new_file() const { return true; }
173 // - Whether the filesystem supports directory modification times
have_directory_mtimes()174 virtual bool have_directory_mtimes() const { return true; }
175 // - Whether some directory tree deletion tests may fail randomly
have_flaky_directory_tree_deletion()176 virtual bool have_flaky_directory_tree_deletion() const { return false; }
177 // - Whether the filesystem stores some metadata alongside files
have_file_metadata()178 virtual bool have_file_metadata() const { return false; }
179
180 void TestEmpty(FileSystem* fs);
181 void TestNormalizePath(FileSystem* fs);
182 void TestCreateDir(FileSystem* fs);
183 void TestDeleteDir(FileSystem* fs);
184 void TestDeleteDirContents(FileSystem* fs);
185 void TestDeleteRootDirContents(FileSystem* fs);
186 void TestDeleteFile(FileSystem* fs);
187 void TestDeleteFiles(FileSystem* fs);
188 void TestMoveFile(FileSystem* fs);
189 void TestMoveDir(FileSystem* fs);
190 void TestCopyFile(FileSystem* fs);
191 void TestGetFileInfo(FileSystem* fs);
192 void TestGetFileInfoVector(FileSystem* fs);
193 void TestGetFileInfoSelector(FileSystem* fs);
194 void TestGetFileInfoSelectorWithRecursion(FileSystem* fs);
195 void TestGetFileInfoAsync(FileSystem* fs);
196 void TestGetFileInfoGenerator(FileSystem* fs);
197 void TestOpenOutputStream(FileSystem* fs);
198 void TestOpenAppendStream(FileSystem* fs);
199 void TestOpenInputStream(FileSystem* fs);
200 void TestOpenInputStreamWithFileInfo(FileSystem* fs);
201 void TestOpenInputStreamAsync(FileSystem* fs);
202 void TestOpenInputFile(FileSystem* fs);
203 void TestOpenInputFileWithFileInfo(FileSystem* fs);
204 void TestOpenInputFileAsync(FileSystem* fs);
205 void TestSpecialChars(FileSystem* fs);
206 };
207
208 #define GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, NAME) \
209 TEST_MACRO(TEST_CLASS, NAME) { this->Test##NAME(); }
210
211 #define GENERIC_FS_TEST_FUNCTIONS_MACROS(TEST_MACRO, TEST_CLASS) \
212 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, Empty) \
213 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, NormalizePath) \
214 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, CreateDir) \
215 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteDir) \
216 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteDirContents) \
217 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteRootDirContents) \
218 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteFile) \
219 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteFiles) \
220 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, MoveFile) \
221 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, MoveDir) \
222 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, CopyFile) \
223 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfo) \
224 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoVector) \
225 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoSelector) \
226 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoSelectorWithRecursion) \
227 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoAsync) \
228 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoGenerator) \
229 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenOutputStream) \
230 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenAppendStream) \
231 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputStream) \
232 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputStreamWithFileInfo) \
233 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputStreamAsync) \
234 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputFile) \
235 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputFileWithFileInfo) \
236 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputFileAsync) \
237 GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, SpecialChars)
238
239 #define GENERIC_FS_TEST_FUNCTIONS(TEST_CLASS) \
240 GENERIC_FS_TEST_FUNCTIONS_MACROS(TEST_F, TEST_CLASS)
241
242 #define GENERIC_FS_TYPED_TEST_FUNCTIONS(TEST_CLASS) \
243 GENERIC_FS_TEST_FUNCTIONS_MACROS(TYPED_TEST, TEST_CLASS)
244
245 } // namespace fs
246 } // namespace arrow
247