1 use std::cmp::Reverse; 2 use std::collections::HashMap; 3 use std::fs::{self, File}; 4 use std::io::{self, Error, ErrorKind, Read, Write}; 5 use std::path::{Path, PathBuf}; 6 use std::sync::{Arc, Mutex}; 7 use std::time::SystemTime; 8 9 use priority_queue::PriorityQueue; 10 11 use crate::authentication::Credentials; 12 use crate::spotify_id::FileId; 13 14 /// Some kind of data structure that holds some paths, the size of these files and a timestamp. 15 /// It keeps track of the file sizes and is able to pop the path with the oldest timestamp if 16 /// a given limit is exceeded. 17 struct SizeLimiter { 18 queue: PriorityQueue<PathBuf, Reverse<SystemTime>>, 19 sizes: HashMap<PathBuf, u64>, 20 size_limit: u64, 21 in_use: u64, 22 } 23 24 impl SizeLimiter { 25 /// Creates a new instance with the given size limit. new(limit: u64) -> Self26 fn new(limit: u64) -> Self { 27 Self { 28 queue: PriorityQueue::new(), 29 sizes: HashMap::new(), 30 size_limit: limit, 31 in_use: 0, 32 } 33 } 34 35 /// Adds an entry to this data structure. 36 /// 37 /// If this file is already contained, it will be updated accordingly. add(&mut self, file: &Path, size: u64, accessed: SystemTime)38 fn add(&mut self, file: &Path, size: u64, accessed: SystemTime) { 39 self.in_use += size; 40 self.queue.push(file.to_owned(), Reverse(accessed)); 41 if let Some(old_size) = self.sizes.insert(file.to_owned(), size) { 42 // It's important that decreasing happens after 43 // increasing the size, to prevent an overflow. 44 self.in_use -= old_size; 45 } 46 } 47 48 /// Returns true if the limit is exceeded. exceeds_limit(&self) -> bool49 fn exceeds_limit(&self) -> bool { 50 self.in_use > self.size_limit 51 } 52 53 /// Returns the least recently accessed file if the size of the cache exceeds 54 /// the limit. 55 /// 56 /// The entry is removed from the data structure, but the caller is responsible 57 /// to delete the file in the file system. pop(&mut self) -> Option<PathBuf>58 fn pop(&mut self) -> Option<PathBuf> { 59 if self.exceeds_limit() { 60 let (next, _) = self 61 .queue 62 .pop() 63 .expect("in_use was > 0, so the queue should have contained an item."); 64 let size = self 65 .sizes 66 .remove(&next) 67 .expect("`queue` and `sizes` should have the same keys."); 68 self.in_use -= size; 69 Some(next) 70 } else { 71 None 72 } 73 } 74 75 /// Updates the timestamp of an existing element. Returns `true` if the item did exist. update(&mut self, file: &Path, access_time: SystemTime) -> bool76 fn update(&mut self, file: &Path, access_time: SystemTime) -> bool { 77 self.queue 78 .change_priority(file, Reverse(access_time)) 79 .is_some() 80 } 81 82 /// Removes an element with the specified path. Returns `true` if the item did exist. remove(&mut self, file: &Path) -> bool83 fn remove(&mut self, file: &Path) -> bool { 84 if self.queue.remove(file).is_none() { 85 return false; 86 } 87 88 let size = self 89 .sizes 90 .remove(file) 91 .expect("`queue` and `sizes` should have the same keys."); 92 self.in_use -= size; 93 94 true 95 } 96 } 97 98 struct FsSizeLimiter { 99 limiter: Mutex<SizeLimiter>, 100 } 101 102 impl FsSizeLimiter { 103 /// Returns access time and file size of a given path. get_metadata(file: &Path) -> io::Result<(SystemTime, u64)>104 fn get_metadata(file: &Path) -> io::Result<(SystemTime, u64)> { 105 let metadata = file.metadata()?; 106 107 // The first of the following timestamps which is available will be chosen as access time: 108 // 1. Access time 109 // 2. Modification time 110 // 3. Creation time 111 // 4. Current time 112 let access_time = metadata 113 .accessed() 114 .or_else(|_| metadata.modified()) 115 .or_else(|_| metadata.created()) 116 .unwrap_or_else(|_| SystemTime::now()); 117 118 let size = metadata.len(); 119 120 Ok((access_time, size)) 121 } 122 123 /// Recursively search a directory for files and add them to the `limiter` struct. init_dir(limiter: &mut SizeLimiter, path: &Path)124 fn init_dir(limiter: &mut SizeLimiter, path: &Path) { 125 let list_dir = match fs::read_dir(path) { 126 Ok(list_dir) => list_dir, 127 Err(e) => { 128 warn!("Could not read directory {:?} in cache dir: {}", path, e); 129 return; 130 } 131 }; 132 133 for entry in list_dir { 134 let entry = match entry { 135 Ok(entry) => entry, 136 Err(e) => { 137 warn!("Could not directory {:?} in cache dir: {}", path, e); 138 return; 139 } 140 }; 141 142 match entry.file_type() { 143 Ok(file_type) if file_type.is_dir() || file_type.is_symlink() => { 144 Self::init_dir(limiter, &entry.path()) 145 } 146 Ok(file_type) if file_type.is_file() => { 147 let path = entry.path(); 148 match Self::get_metadata(&path) { 149 Ok((access_time, size)) => { 150 limiter.add(&path, size, access_time); 151 } 152 Err(e) => { 153 warn!("Could not read file {:?} in cache dir: {}", path, e) 154 } 155 } 156 } 157 Ok(ft) => { 158 warn!( 159 "File {:?} in cache dir has unsupported type {:?}", 160 entry.path(), 161 ft 162 ) 163 } 164 Err(e) => { 165 warn!( 166 "Could not get type of file {:?} in cache dir: {}", 167 entry.path(), 168 e 169 ) 170 } 171 }; 172 } 173 } 174 add(&self, file: &Path, size: u64)175 fn add(&self, file: &Path, size: u64) { 176 self.limiter 177 .lock() 178 .unwrap() 179 .add(file, size, SystemTime::now()); 180 } 181 touch(&self, file: &Path) -> bool182 fn touch(&self, file: &Path) -> bool { 183 self.limiter.lock().unwrap().update(file, SystemTime::now()) 184 } 185 remove(&self, file: &Path)186 fn remove(&self, file: &Path) { 187 self.limiter.lock().unwrap().remove(file); 188 } 189 prune_internal<F: FnMut() -> Option<PathBuf>>(mut pop: F)190 fn prune_internal<F: FnMut() -> Option<PathBuf>>(mut pop: F) { 191 let mut first = true; 192 let mut count = 0; 193 194 while let Some(file) = pop() { 195 if first { 196 debug!("Cache dir exceeds limit, removing least recently used files."); 197 first = false; 198 } 199 200 if let Err(e) = fs::remove_file(&file) { 201 warn!("Could not remove file {:?} from cache dir: {}", file, e); 202 } else { 203 count += 1; 204 } 205 } 206 207 if count > 0 { 208 info!("Removed {} cache files.", count); 209 } 210 } 211 prune(&self)212 fn prune(&self) { 213 Self::prune_internal(|| self.limiter.lock().unwrap().pop()) 214 } 215 new(path: &Path, limit: u64) -> Self216 fn new(path: &Path, limit: u64) -> Self { 217 let mut limiter = SizeLimiter::new(limit); 218 219 Self::init_dir(&mut limiter, path); 220 Self::prune_internal(|| limiter.pop()); 221 222 Self { 223 limiter: Mutex::new(limiter), 224 } 225 } 226 } 227 228 /// A cache for volume, credentials and audio files. 229 #[derive(Clone)] 230 pub struct Cache { 231 credentials_location: Option<PathBuf>, 232 volume_location: Option<PathBuf>, 233 audio_location: Option<PathBuf>, 234 size_limiter: Option<Arc<FsSizeLimiter>>, 235 } 236 237 pub struct RemoveFileError(()); 238 239 impl Cache { new<P: AsRef<Path>>( system_location: Option<P>, audio_location: Option<P>, size_limit: Option<u64>, ) -> io::Result<Self>240 pub fn new<P: AsRef<Path>>( 241 system_location: Option<P>, 242 audio_location: Option<P>, 243 size_limit: Option<u64>, 244 ) -> io::Result<Self> { 245 if let Some(location) = &system_location { 246 fs::create_dir_all(location)?; 247 } 248 249 let mut size_limiter = None; 250 251 if let Some(location) = &audio_location { 252 fs::create_dir_all(location)?; 253 if let Some(limit) = size_limit { 254 let limiter = FsSizeLimiter::new(location.as_ref(), limit); 255 size_limiter = Some(Arc::new(limiter)); 256 } 257 } 258 259 let audio_location = audio_location.map(|p| p.as_ref().to_owned()); 260 let volume_location = system_location.as_ref().map(|p| p.as_ref().join("volume")); 261 let credentials_location = system_location 262 .as_ref() 263 .map(|p| p.as_ref().join("credentials.json")); 264 265 let cache = Cache { 266 credentials_location, 267 volume_location, 268 audio_location, 269 size_limiter, 270 }; 271 272 Ok(cache) 273 } 274 credentials(&self) -> Option<Credentials>275 pub fn credentials(&self) -> Option<Credentials> { 276 let location = self.credentials_location.as_ref()?; 277 278 // This closure is just convencience to enable the question mark operator 279 let read = || { 280 let mut file = File::open(location)?; 281 let mut contents = String::new(); 282 file.read_to_string(&mut contents)?; 283 serde_json::from_str(&contents).map_err(|e| Error::new(ErrorKind::InvalidData, e)) 284 }; 285 286 match read() { 287 Ok(c) => Some(c), 288 Err(e) => { 289 // If the file did not exist, the file was probably not written 290 // before. Otherwise, log the error. 291 if e.kind() != ErrorKind::NotFound { 292 warn!("Error reading credentials from cache: {}", e); 293 } 294 None 295 } 296 } 297 } 298 save_credentials(&self, cred: &Credentials)299 pub fn save_credentials(&self, cred: &Credentials) { 300 if let Some(location) = &self.credentials_location { 301 let result = File::create(location).and_then(|mut file| { 302 let data = serde_json::to_string(cred)?; 303 write!(file, "{}", data) 304 }); 305 306 if let Err(e) = result { 307 warn!("Cannot save credentials to cache: {}", e) 308 } 309 } 310 } 311 volume(&self) -> Option<u16>312 pub fn volume(&self) -> Option<u16> { 313 let location = self.volume_location.as_ref()?; 314 315 let read = || { 316 let mut file = File::open(location)?; 317 let mut contents = String::new(); 318 file.read_to_string(&mut contents)?; 319 contents 320 .parse() 321 .map_err(|e| Error::new(ErrorKind::InvalidData, e)) 322 }; 323 324 match read() { 325 Ok(v) => Some(v), 326 Err(e) => { 327 if e.kind() != ErrorKind::NotFound { 328 warn!("Error reading volume from cache: {}", e); 329 } 330 None 331 } 332 } 333 } 334 save_volume(&self, volume: u16)335 pub fn save_volume(&self, volume: u16) { 336 if let Some(ref location) = self.volume_location { 337 let result = File::create(location).and_then(|mut file| write!(file, "{}", volume)); 338 if let Err(e) = result { 339 warn!("Cannot save volume to cache: {}", e); 340 } 341 } 342 } 343 file_path(&self, file: FileId) -> Option<PathBuf>344 fn file_path(&self, file: FileId) -> Option<PathBuf> { 345 self.audio_location.as_ref().map(|location| { 346 let name = file.to_base16(); 347 let mut path = location.join(&name[0..2]); 348 path.push(&name[2..]); 349 path 350 }) 351 } 352 file(&self, file: FileId) -> Option<File>353 pub fn file(&self, file: FileId) -> Option<File> { 354 let path = self.file_path(file)?; 355 match File::open(&path) { 356 Ok(file) => { 357 if let Some(limiter) = self.size_limiter.as_deref() { 358 limiter.touch(&path); 359 } 360 Some(file) 361 } 362 Err(e) => { 363 if e.kind() != ErrorKind::NotFound { 364 warn!("Error reading file from cache: {}", e) 365 } 366 None 367 } 368 } 369 } 370 save_file<F: Read>(&self, file: FileId, contents: &mut F)371 pub fn save_file<F: Read>(&self, file: FileId, contents: &mut F) { 372 let path = if let Some(path) = self.file_path(file) { 373 path 374 } else { 375 return; 376 }; 377 let parent = path.parent().unwrap(); 378 379 let result = fs::create_dir_all(parent) 380 .and_then(|_| File::create(&path)) 381 .and_then(|mut file| io::copy(contents, &mut file)); 382 383 if let Ok(size) = result { 384 if let Some(limiter) = self.size_limiter.as_deref() { 385 limiter.add(&path, size); 386 limiter.prune(); 387 } 388 } 389 } 390 remove_file(&self, file: FileId) -> Result<(), RemoveFileError>391 pub fn remove_file(&self, file: FileId) -> Result<(), RemoveFileError> { 392 let path = self.file_path(file).ok_or(RemoveFileError(()))?; 393 394 if let Err(err) = fs::remove_file(&path) { 395 warn!("Unable to remove file from cache: {}", err); 396 Err(RemoveFileError(())) 397 } else { 398 if let Some(limiter) = self.size_limiter.as_deref() { 399 limiter.remove(&path); 400 } 401 Ok(()) 402 } 403 } 404 } 405 406 #[cfg(test)] 407 mod test { 408 use super::*; 409 use std::time::Duration; 410 ordered_time(v: u64) -> SystemTime411 fn ordered_time(v: u64) -> SystemTime { 412 SystemTime::UNIX_EPOCH + Duration::from_secs(v) 413 } 414 415 #[test] test_size_limiter()416 fn test_size_limiter() { 417 let mut limiter = SizeLimiter::new(1000); 418 419 limiter.add(Path::new("a"), 500, ordered_time(2)); 420 limiter.add(Path::new("b"), 500, ordered_time(1)); 421 422 // b (500) -> a (500) => sum: 1000 <= 1000 423 assert!(!limiter.exceeds_limit()); 424 assert_eq!(limiter.pop(), None); 425 426 limiter.add(Path::new("c"), 1000, ordered_time(3)); 427 428 // b (500) -> a (500) -> c (1000) => sum: 2000 > 1000 429 assert!(limiter.exceeds_limit()); 430 assert_eq!(limiter.pop().as_deref(), Some(Path::new("b"))); 431 // a (500) -> c (1000) => sum: 1500 > 1000 432 assert_eq!(limiter.pop().as_deref(), Some(Path::new("a"))); 433 // c (1000) => sum: 1000 <= 1000 434 assert_eq!(limiter.pop().as_deref(), None); 435 436 limiter.add(Path::new("d"), 5, ordered_time(2)); 437 // d (5) -> c (1000) => sum: 1005 > 1000 438 assert_eq!(limiter.pop().as_deref(), Some(Path::new("d"))); 439 // c (1000) => sum: 1000 <= 1000 440 assert_eq!(limiter.pop().as_deref(), None); 441 442 // Test updating 443 444 limiter.add(Path::new("e"), 500, ordered_time(3)); 445 // c (1000) -> e (500) => sum: 1500 > 1000 446 assert!(limiter.update(Path::new("c"), ordered_time(4))); 447 // e (500) -> c (1000) => sum: 1500 > 1000 448 assert_eq!(limiter.pop().as_deref(), Some(Path::new("e"))); 449 // c (1000) => sum: 1000 <= 1000 450 451 // Test removing 452 limiter.add(Path::new("f"), 500, ordered_time(2)); 453 assert!(limiter.remove(Path::new("c"))); 454 assert!(!limiter.exceeds_limit()); 455 } 456 } 457