1 use super::*;
2 use crate::pristine::{ArcTxn, InodeMetadata};
3 use canonical_path::{CanonicalPath, CanonicalPathBuf};
4 use ignore::WalkBuilder;
5 use std::borrow::Cow;
6 use std::path::{Path, PathBuf};
7 
8 #[derive(Clone)]
9 pub struct FileSystem {
10     root: PathBuf,
11 }
12 
filter_ignore(root_: &CanonicalPath, path: &CanonicalPath, is_dir: bool) -> bool13 pub fn filter_ignore(root_: &CanonicalPath, path: &CanonicalPath, is_dir: bool) -> bool {
14     debug!("path = {:?} root = {:?}", path, root_);
15     if let Ok(suffix) = path.as_path().strip_prefix(root_.as_path()) {
16         debug!("suffix = {:?}", suffix);
17         let mut root = root_.as_path().to_path_buf();
18         let mut ignore = ignore::gitignore::GitignoreBuilder::new(&root);
19         let mut add_root = |root: &mut PathBuf| {
20             ignore.add_line(None, crate::DOT_DIR).unwrap();
21             root.push(".ignore");
22             ignore.add(&root);
23             root.pop();
24             root.push(".gitignore");
25             ignore.add(&root);
26             root.pop();
27         };
28         add_root(&mut root);
29         for c in suffix.components() {
30             root.push(c);
31             add_root(&mut root);
32         }
33         if let Ok(ig) = ignore.build() {
34             let m = ig.matched(suffix, is_dir);
35             debug!("m = {:?}", m);
36             return !m.is_ignore();
37         }
38     }
39     false
40 }
41 
42 /// From a path on the filesystem, return the canonical path (a `PathBuf`), and a
43 /// prefix relative to the root of the repository (a `String`).
get_prefix( repo_path: Option<&CanonicalPath>, prefix: &Path, ) -> Result<(canonical_path::CanonicalPathBuf, String), std::io::Error>44 pub fn get_prefix(
45     repo_path: Option<&CanonicalPath>,
46     prefix: &Path,
47 ) -> Result<(canonical_path::CanonicalPathBuf, String), std::io::Error> {
48     let mut p = String::new();
49     let repo = if let Some(repo) = repo_path {
50         Cow::Borrowed(repo)
51     } else {
52         Cow::Owned(canonical_path::CanonicalPathBuf::canonicalize(
53             std::env::current_dir()?,
54         )?)
55     };
56     debug!("get prefix {:?} {:?}", repo, prefix);
57     let prefix_ = CanonicalPathBuf::canonicalize(&repo.as_path().join(&prefix))?;
58     debug!("get prefix {:?}", prefix_);
59     if let Ok(prefix) = prefix_.as_path().strip_prefix(repo.as_path()) {
60         for c in prefix.components() {
61             if !p.is_empty() {
62                 p.push('/');
63             }
64             let c: &std::path::Path = c.as_ref();
65             p.push_str(&c.to_string_lossy())
66         }
67     }
68     Ok((prefix_, p))
69 }
70 
71 #[derive(Debug, Error)]
72 pub enum AddError<T: std::error::Error + 'static> {
73     #[error(transparent)]
74     Ignore(#[from] ignore::Error),
75     #[error(transparent)]
76     Io(#[from] std::io::Error),
77     #[error(transparent)]
78     Fs(#[from] crate::fs::FsError<T>),
79 }
80 
81 #[derive(Debug, Error)]
82 pub enum Error<C: std::error::Error + 'static, T: std::error::Error + 'static> {
83     #[error(transparent)]
84     Add(#[from] AddError<T>),
85     #[error(transparent)]
86     Record(#[from] crate::record::RecordError<C, std::io::Error, T>),
87     #[error(transparent)]
88     Txn(#[from] T),
89 }
90 
91 pub struct Untracked {
92     join: Option<std::thread::JoinHandle<Result<(), std::io::Error>>>,
93     receiver: std::sync::mpsc::Receiver<(PathBuf, bool)>,
94 }
95 
96 impl Iterator for Untracked {
97     type Item = Result<(PathBuf, bool), std::io::Error>;
next(&mut self) -> Option<Self::Item>98     fn next(&mut self) -> Option<Self::Item> {
99         if let Ok(x) = self.receiver.recv() {
100             return Some(Ok(x));
101         } else if let Some(j) = self.join.take() {
102             if let Ok(Err(e)) = j.join() {
103                 return Some(Err(e));
104             }
105         }
106         None
107     }
108 }
109 
110 impl FileSystem {
from_root<P: AsRef<Path>>(root: P) -> Self111     pub fn from_root<P: AsRef<Path>>(root: P) -> Self {
112         FileSystem {
113             root: root.as_ref().to_path_buf(),
114         }
115     }
116 
record_prefixes< T: crate::MutTxnTExt + crate::TxnTExt + Send + Sync + 'static, C: crate::changestore::ChangeStore + Clone + Send + 'static, P: AsRef<Path>, >( &self, txn: ArcTxn<T>, channel: crate::pristine::ChannelRef<T>, changes: &C, state: &mut crate::RecordBuilder, repo_path: CanonicalPathBuf, prefixes: &[P], threads: usize, salt: u64, ) -> Result<(), Error<C::Error, T::GraphError>> where T::Channel: Send + Sync,117     pub fn record_prefixes<
118         T: crate::MutTxnTExt + crate::TxnTExt + Send + Sync + 'static,
119         C: crate::changestore::ChangeStore + Clone + Send + 'static,
120         P: AsRef<Path>,
121     >(
122         &self,
123         txn: ArcTxn<T>,
124         channel: crate::pristine::ChannelRef<T>,
125         changes: &C,
126         state: &mut crate::RecordBuilder,
127         repo_path: CanonicalPathBuf,
128         prefixes: &[P],
129         threads: usize,
130         salt: u64,
131     ) -> Result<(), Error<C::Error, T::GraphError>>
132     where
133         T::Channel: Send + Sync,
134     {
135         for prefix in prefixes.iter() {
136             self.clone().record_prefix(
137                 txn.clone(),
138                 channel.clone(),
139                 changes,
140                 state,
141                 repo_path.clone(),
142                 prefix.as_ref(),
143                 threads,
144                 salt,
145             )?
146         }
147         if prefixes.is_empty() {
148             self.record_prefix(
149                 txn,
150                 channel,
151                 changes,
152                 state,
153                 repo_path.clone(),
154                 Path::new(""),
155                 threads,
156                 salt,
157             )?
158         }
159         Ok(())
160     }
161 
add_prefix_rec<T: crate::MutTxnTExt + crate::TxnTExt>( &self, txn: &ArcTxn<T>, repo_path: CanonicalPathBuf, full: CanonicalPathBuf, threads: usize, salt: u64, ) -> Result<(), AddError<T::GraphError>>162     pub fn add_prefix_rec<T: crate::MutTxnTExt + crate::TxnTExt>(
163         &self,
164         txn: &ArcTxn<T>,
165         repo_path: CanonicalPathBuf,
166         full: CanonicalPathBuf,
167         threads: usize,
168         salt: u64,
169     ) -> Result<(), AddError<T::GraphError>> {
170         let mut txn = txn.write();
171         for p in self.iterate_prefix_rec(repo_path.clone(), full.clone(), threads)? {
172             let (path, is_dir) = p?;
173             info!("Adding {:?}", path);
174             use path_slash::PathExt;
175             let path_str = path.to_slash_lossy();
176             if path_str.is_empty() || path_str == "." {
177                 continue;
178             }
179             match txn.add(&path_str, is_dir, salt) {
180                 Ok(_) => {}
181                 Err(crate::fs::FsError::AlreadyInRepo(_)) => {}
182                 Err(e) => return Err(e.into()),
183             }
184         }
185         Ok(())
186     }
187 
iterate_prefix_rec( &self, repo_path: CanonicalPathBuf, full: CanonicalPathBuf, threads: usize, ) -> Result<Untracked, std::io::Error>188     pub fn iterate_prefix_rec(
189         &self,
190         repo_path: CanonicalPathBuf,
191         full: CanonicalPathBuf,
192         threads: usize,
193     ) -> Result<Untracked, std::io::Error> {
194         debug!("full = {:?}", full);
195         let meta = std::fs::metadata(&full)?;
196         debug!("meta = {:?}", meta);
197         let (sender, receiver) = std::sync::mpsc::sync_channel(100);
198 
199         debug!("{:?}", full.as_path().strip_prefix(repo_path.as_path()));
200         if !filter_ignore(
201             &repo_path.as_canonical_path(),
202             &full.as_canonical_path(),
203             meta.is_dir(),
204         ) {
205             return Ok(Untracked {
206                 join: None,
207                 receiver,
208             });
209         }
210         let t = std::thread::spawn(move || -> Result<(), std::io::Error> {
211             if meta.is_dir() {
212                 let mut walk = WalkBuilder::new(&full);
213                 walk.ignore(true)
214                     .git_ignore(true)
215                     .hidden(false)
216                     .filter_entry(|p| {
217                         debug!("p.file_name = {:?}", p.file_name());
218                         p.file_name() != crate::DOT_DIR
219                     })
220                     .threads((threads - 1).max(1));
221                 walk.build_parallel().run(|| {
222                     Box::new(|entry| {
223                         let entry: ignore::DirEntry = if let Ok(entry) = entry {
224                             entry
225                         } else {
226                             return ignore::WalkState::Quit;
227                         };
228                         let p = entry.path();
229                         if let Some(p) = p.file_name() {
230                             if let Some(p) = p.to_str() {
231                                 if p.ends_with("~") || (p.starts_with("#") && p.ends_with("#")) {
232                                     return ignore::WalkState::Skip;
233                                 }
234                             }
235                         }
236                         debug!("entry path = {:?} {:?}", entry.path(), repo_path);
237                         if let Ok(entry_path) = CanonicalPathBuf::canonicalize(entry.path()) {
238                             if let Ok(path) = entry_path.as_path().strip_prefix(&repo_path) {
239                                 let is_dir = entry.file_type().unwrap().is_dir();
240                                 if sender.send((path.to_path_buf(), is_dir)).is_err() {
241                                     return ignore::WalkState::Quit;
242                                 }
243                             } else {
244                                 debug!("entry = {:?}", entry.path());
245                             }
246                         }
247                         ignore::WalkState::Continue
248                     })
249                 })
250             } else {
251                 debug!("filter_ignore ok");
252                 let path = full.as_path().strip_prefix(&repo_path.as_path()).unwrap();
253                 sender.send((path.to_path_buf(), false)).unwrap();
254             }
255             Ok(())
256         });
257         Ok(Untracked {
258             join: Some(t),
259             receiver,
260         })
261     }
262 
record_prefix< T: crate::MutTxnTExt + crate::TxnTExt + Send + Sync + 'static, C: crate::changestore::ChangeStore + Clone + Send + 'static, >( &self, txn: ArcTxn<T>, channel: crate::pristine::ChannelRef<T>, changes: &C, state: &mut crate::RecordBuilder, repo_path: CanonicalPathBuf, prefix: &Path, threads: usize, salt: u64, ) -> Result<(), Error<C::Error, T::GraphError>> where T::Channel: Send + Sync,263     pub fn record_prefix<
264         T: crate::MutTxnTExt + crate::TxnTExt + Send + Sync + 'static,
265         C: crate::changestore::ChangeStore + Clone + Send + 'static,
266     >(
267         &self,
268         txn: ArcTxn<T>,
269         channel: crate::pristine::ChannelRef<T>,
270         changes: &C,
271         state: &mut crate::RecordBuilder,
272         repo_path: CanonicalPathBuf,
273         prefix: &Path,
274         threads: usize,
275         salt: u64,
276     ) -> Result<(), Error<C::Error, T::GraphError>>
277     where
278         T::Channel: Send + Sync,
279     {
280         let (full, prefix) = get_prefix(Some(repo_path.as_ref()), prefix).map_err(AddError::Io)?;
281         {
282             let path = if let Ok(path) = full.as_path().strip_prefix(&repo_path.as_path()) {
283                 path
284             } else {
285                 return Ok(());
286             };
287             use path_slash::PathExt;
288             let path_str = path.to_slash_lossy();
289             if !txn.read().is_tracked(&path_str)? {
290                 self.add_prefix_rec(&txn, repo_path, full, threads, salt)?;
291             }
292         }
293         debug!("recording from prefix {:?}", prefix);
294         state.record(
295             txn.clone(),
296             crate::Algorithm::default(),
297             &crate::diff::DEFAULT_SEPARATOR,
298             channel,
299             self,
300             changes,
301             &prefix,
302             1,
303         )?;
304         debug!("recorded");
305         Ok(())
306     }
307 
path(&self, file: &str) -> PathBuf308     fn path(&self, file: &str) -> PathBuf {
309         let mut path = self.root.clone();
310         path.extend(crate::path::components(file));
311         path
312     }
313 }
314 
315 impl WorkingCopyRead for FileSystem {
316     type Error = std::io::Error;
file_metadata(&self, file: &str) -> Result<InodeMetadata, Self::Error>317     fn file_metadata(&self, file: &str) -> Result<InodeMetadata, Self::Error> {
318         debug!("metadata {:?}", file);
319         let attr = std::fs::metadata(&self.path(file))?;
320         let permissions = permissions(&attr).unwrap_or(0o700);
321         debug!("permissions = {:?}", permissions);
322         Ok(InodeMetadata::new(permissions & 0o100, attr.is_dir()))
323     }
read_file(&self, file: &str, buffer: &mut Vec<u8>) -> Result<(), Self::Error>324     fn read_file(&self, file: &str, buffer: &mut Vec<u8>) -> Result<(), Self::Error> {
325         use std::io::Read;
326         debug!("read_file {:?}", file);
327         let mut f = std::fs::File::open(&self.path(file))?;
328         f.read_to_end(buffer)?;
329         Ok(())
330     }
331 
332     #[cfg(not(unix))]
modified_time(&self, file: &str) -> Result<std::time::SystemTime, Self::Error>333     fn modified_time(&self, file: &str) -> Result<std::time::SystemTime, Self::Error> {
334         debug!("modified_time {:?}", file);
335         let attr = std::fs::metadata(&self.path(file))?;
336         Ok(attr.modified()?)
337     }
338 
339     #[cfg(unix)]
modified_time(&self, file: &str) -> Result<std::time::SystemTime, Self::Error>340     fn modified_time(&self, file: &str) -> Result<std::time::SystemTime, Self::Error> {
341         debug!("modified_time {:?}", file);
342         use std::os::unix::fs::MetadataExt;
343         let attr = std::fs::metadata(&self.path(file))?;
344         let ctime =
345             std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(attr.ctime() as u64);
346         Ok(attr.modified()?.min(ctime))
347     }
348 }
349 
350 impl WorkingCopy for FileSystem {
create_dir_all(&self, file: &str) -> Result<(), Self::Error>351     fn create_dir_all(&self, file: &str) -> Result<(), Self::Error> {
352         debug!("create_dir_all {:?}", file);
353         Ok(std::fs::create_dir_all(&self.path(file))?)
354     }
355 
remove_path(&self, path: &str, rec: bool) -> Result<(), Self::Error>356     fn remove_path(&self, path: &str, rec: bool) -> Result<(), Self::Error> {
357         debug!("remove_path {:?}", path);
358         let path = self.path(path);
359         if let Ok(meta) = std::fs::metadata(&path) {
360             if let Err(e) = if meta.is_dir() {
361                 if rec {
362                     std::fs::remove_dir_all(&path)
363                 } else {
364                     std::fs::remove_dir(&path)
365                 }
366             } else {
367                 std::fs::remove_file(&path)
368             } {
369                 info!("while deleting {:?}: {:?}", path, e);
370             }
371         }
372         Ok(())
373     }
rename(&self, former: &str, new: &str) -> Result<(), Self::Error>374     fn rename(&self, former: &str, new: &str) -> Result<(), Self::Error> {
375         debug!("rename {:?} {:?}", former, new);
376         let former = self.path(former);
377         let new = self.path(new);
378         if let Some(p) = new.parent() {
379             std::fs::create_dir_all(p)?
380         }
381         std::fs::rename(&former, &new)?;
382         Ok(())
383     }
384     #[cfg(not(windows))]
set_permissions(&self, name: &str, permissions: u16) -> Result<(), Self::Error>385     fn set_permissions(&self, name: &str, permissions: u16) -> Result<(), Self::Error> {
386         use std::os::unix::fs::PermissionsExt;
387         let name = self.path(name);
388         debug!("set_permissions: {:?}", name);
389         let metadata = std::fs::metadata(&name)?;
390         let mut current = metadata.permissions();
391         debug!(
392             "setting mode for {:?} to {:?} (currently {:?})",
393             name, permissions, current
394         );
395         if permissions & 0o100 != 0 {
396             current.set_mode(current.mode() | 0o100);
397         } else {
398             current.set_mode(current.mode() & ((!0o777) | 0o666));
399         }
400         debug!("setting {:?}", current);
401         std::fs::set_permissions(name, current)?;
402         debug!("set");
403         Ok(())
404     }
405     #[cfg(windows)]
set_permissions(&self, _name: &str, _permissions: u16) -> Result<(), Self::Error>406     fn set_permissions(&self, _name: &str, _permissions: u16) -> Result<(), Self::Error> {
407         Ok(())
408     }
409 
410     type Writer = std::io::BufWriter<std::fs::File>;
write_file(&self, file: &str) -> Result<Self::Writer, Self::Error>411     fn write_file(&self, file: &str) -> Result<Self::Writer, Self::Error> {
412         let path = self.path(file);
413         debug!("path = {:?}", path);
414         if let Some(p) = path.parent() {
415             std::fs::create_dir_all(p).unwrap_or(())
416         }
417         debug!("write_file: dir created");
418         std::fs::remove_file(&path).unwrap_or(());
419         let file = std::io::BufWriter::new(std::fs::File::create(&path)?);
420         debug!("file");
421         Ok(file)
422     }
423 }
424 
425 #[cfg(not(windows))]
permissions(attr: &std::fs::Metadata) -> Option<usize>426 fn permissions(attr: &std::fs::Metadata) -> Option<usize> {
427     use std::os::unix::fs::PermissionsExt;
428     Some(attr.permissions().mode() as usize)
429 }
430 #[cfg(windows)]
permissions(_: &std::fs::Metadata) -> Option<usize>431 fn permissions(_: &std::fs::Metadata) -> Option<usize> {
432     None
433 }
434