1package main 2 3import ( 4 "bytes" 5 "fmt" 6 "io" 7 "os" 8 "path/filepath" 9 "strconv" 10 "strings" 11 "sync" 12 13 "github.com/restic/restic/internal/debug" 14 "github.com/restic/restic/internal/errors" 15 "github.com/restic/restic/internal/filter" 16 "github.com/restic/restic/internal/fs" 17 "github.com/restic/restic/internal/repository" 18) 19 20type rejectionCache struct { 21 m map[string]bool 22 mtx sync.Mutex 23} 24 25// Lock locks the mutex in rc. 26func (rc *rejectionCache) Lock() { 27 if rc != nil { 28 rc.mtx.Lock() 29 } 30} 31 32// Unlock unlocks the mutex in rc. 33func (rc *rejectionCache) Unlock() { 34 if rc != nil { 35 rc.mtx.Unlock() 36 } 37} 38 39// Get returns the last stored value for dir and a second boolean that 40// indicates whether that value was actually written to the cache. It is the 41// callers responsibility to call rc.Lock and rc.Unlock before using this 42// method, otherwise data races may occur. 43func (rc *rejectionCache) Get(dir string) (bool, bool) { 44 if rc == nil || rc.m == nil { 45 return false, false 46 } 47 v, ok := rc.m[dir] 48 return v, ok 49} 50 51// Store stores a new value for dir. It is the callers responsibility to call 52// rc.Lock and rc.Unlock before using this method, otherwise data races may 53// occur. 54func (rc *rejectionCache) Store(dir string, rejected bool) { 55 if rc == nil { 56 return 57 } 58 if rc.m == nil { 59 rc.m = make(map[string]bool) 60 } 61 rc.m[dir] = rejected 62} 63 64// RejectByNameFunc is a function that takes a filename of a 65// file that would be included in the backup. The function returns true if it 66// should be excluded (rejected) from the backup. 67type RejectByNameFunc func(path string) bool 68 69// RejectFunc is a function that takes a filename and os.FileInfo of a 70// file that would be included in the backup. The function returns true if it 71// should be excluded (rejected) from the backup. 72type RejectFunc func(path string, fi os.FileInfo) bool 73 74// rejectByPattern returns a RejectByNameFunc which rejects files that match 75// one of the patterns. 76func rejectByPattern(patterns []string) RejectByNameFunc { 77 parsedPatterns := filter.ParsePatterns(patterns) 78 return func(item string) bool { 79 matched, err := filter.List(parsedPatterns, item) 80 if err != nil { 81 Warnf("error for exclude pattern: %v", err) 82 } 83 84 if matched { 85 debug.Log("path %q excluded by an exclude pattern", item) 86 return true 87 } 88 89 return false 90 } 91} 92 93// Same as `rejectByPattern` but case insensitive. 94func rejectByInsensitivePattern(patterns []string) RejectByNameFunc { 95 for index, path := range patterns { 96 patterns[index] = strings.ToLower(path) 97 } 98 99 rejFunc := rejectByPattern(patterns) 100 return func(item string) bool { 101 return rejFunc(strings.ToLower(item)) 102 } 103} 104 105// rejectIfPresent returns a RejectByNameFunc which itself returns whether a path 106// should be excluded. The RejectByNameFunc considers a file to be excluded when 107// it resides in a directory with an exclusion file, that is specified by 108// excludeFileSpec in the form "filename[:content]". The returned error is 109// non-nil if the filename component of excludeFileSpec is empty. If rc is 110// non-nil, it is going to be used in the RejectByNameFunc to expedite the evaluation 111// of a directory based on previous visits. 112func rejectIfPresent(excludeFileSpec string) (RejectByNameFunc, error) { 113 if excludeFileSpec == "" { 114 return nil, errors.New("name for exclusion tagfile is empty") 115 } 116 colon := strings.Index(excludeFileSpec, ":") 117 if colon == 0 { 118 return nil, fmt.Errorf("no name for exclusion tagfile provided") 119 } 120 tf, tc := "", "" 121 if colon > 0 { 122 tf = excludeFileSpec[:colon] 123 tc = excludeFileSpec[colon+1:] 124 } else { 125 tf = excludeFileSpec 126 } 127 debug.Log("using %q as exclusion tagfile", tf) 128 rc := &rejectionCache{} 129 fn := func(filename string) bool { 130 return isExcludedByFile(filename, tf, tc, rc) 131 } 132 return fn, nil 133} 134 135// isExcludedByFile interprets filename as a path and returns true if that file 136// is in an excluded directory. A directory is identified as excluded if it contains a 137// tagfile which bears the name specified in tagFilename and starts with 138// header. If rc is non-nil, it is used to expedite the evaluation of a 139// directory based on previous visits. 140func isExcludedByFile(filename, tagFilename, header string, rc *rejectionCache) bool { 141 if tagFilename == "" { 142 return false 143 } 144 dir, base := filepath.Split(filename) 145 if base == tagFilename { 146 return false // do not exclude the tagfile itself 147 } 148 rc.Lock() 149 defer rc.Unlock() 150 151 rejected, visited := rc.Get(dir) 152 if visited { 153 return rejected 154 } 155 rejected = isDirExcludedByFile(dir, tagFilename, header) 156 rc.Store(dir, rejected) 157 return rejected 158} 159 160func isDirExcludedByFile(dir, tagFilename, header string) bool { 161 tf := filepath.Join(dir, tagFilename) 162 _, err := fs.Lstat(tf) 163 if os.IsNotExist(err) { 164 return false 165 } 166 if err != nil { 167 Warnf("could not access exclusion tagfile: %v", err) 168 return false 169 } 170 // when no signature is given, the mere presence of tf is enough reason 171 // to exclude filename 172 if len(header) == 0 { 173 return true 174 } 175 // From this stage, errors mean tagFilename exists but it is malformed. 176 // Warnings will be generated so that the user is informed that the 177 // indented ignore-action is not performed. 178 f, err := os.Open(tf) 179 if err != nil { 180 Warnf("could not open exclusion tagfile: %v", err) 181 return false 182 } 183 defer func() { 184 _ = f.Close() 185 }() 186 buf := make([]byte, len(header)) 187 _, err = io.ReadFull(f, buf) 188 // EOF is handled with a dedicated message, otherwise the warning were too cryptic 189 if err == io.EOF { 190 Warnf("invalid (too short) signature in exclusion tagfile %q\n", tf) 191 return false 192 } 193 if err != nil { 194 Warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err) 195 return false 196 } 197 if !bytes.Equal(buf, []byte(header)) { 198 Warnf("invalid signature in exclusion tagfile %q\n", tf) 199 return false 200 } 201 return true 202} 203 204// DeviceMap is used to track allowed source devices for backup. This is used to 205// check for crossing mount points during backup (for --one-file-system). It 206// maps the name of a source path to its device ID. 207type DeviceMap map[string]uint64 208 209// NewDeviceMap creates a new device map from the list of source paths. 210func NewDeviceMap(allowedSourcePaths []string) (DeviceMap, error) { 211 deviceMap := make(map[string]uint64) 212 213 for _, item := range allowedSourcePaths { 214 item, err := filepath.Abs(filepath.Clean(item)) 215 if err != nil { 216 return nil, err 217 } 218 219 fi, err := fs.Lstat(item) 220 if err != nil { 221 return nil, err 222 } 223 224 id, err := fs.DeviceID(fi) 225 if err != nil { 226 return nil, err 227 } 228 229 deviceMap[item] = id 230 } 231 232 if len(deviceMap) == 0 { 233 return nil, errors.New("zero allowed devices") 234 } 235 236 return deviceMap, nil 237} 238 239// IsAllowed returns true if the path is located on an allowed device. 240func (m DeviceMap) IsAllowed(item string, deviceID uint64) (bool, error) { 241 for dir := item; ; dir = filepath.Dir(dir) { 242 debug.Log("item %v, test dir %v", item, dir) 243 244 // find a parent directory that is on an allowed device (otherwise 245 // we would not traverse the directory at all) 246 allowedID, ok := m[dir] 247 if !ok { 248 if dir == filepath.Dir(dir) { 249 // arrived at root, no allowed device found. this should not happen. 250 break 251 } 252 continue 253 } 254 255 // if the item has a different device ID than the parent directory, 256 // we crossed a file system boundary 257 if allowedID != deviceID { 258 debug.Log("item %v (dir %v) on disallowed device %d", item, dir, deviceID) 259 return false, nil 260 } 261 262 // item is on allowed device, accept it 263 debug.Log("item %v allowed", item) 264 return true, nil 265 } 266 267 return false, fmt.Errorf("item %v (device ID %v) not found, deviceMap: %v", item, deviceID, m) 268} 269 270// rejectByDevice returns a RejectFunc that rejects files which are on a 271// different file systems than the files/dirs in samples. 272func rejectByDevice(samples []string) (RejectFunc, error) { 273 deviceMap, err := NewDeviceMap(samples) 274 if err != nil { 275 return nil, err 276 } 277 debug.Log("allowed devices: %v\n", deviceMap) 278 279 return func(item string, fi os.FileInfo) bool { 280 id, err := fs.DeviceID(fi) 281 if err != nil { 282 // This should never happen because gatherDevices() would have 283 // errored out earlier. If it still does that's a reason to panic. 284 panic(err) 285 } 286 287 allowed, err := deviceMap.IsAllowed(filepath.Clean(item), id) 288 if err != nil { 289 // this should not happen 290 panic(fmt.Sprintf("error checking device ID of %v: %v", item, err)) 291 } 292 293 if allowed { 294 // accept item 295 return false 296 } 297 298 // reject everything except directories 299 if !fi.IsDir() { 300 return true 301 } 302 303 // special case: make sure we keep mountpoints (directories which 304 // contain a mounted file system). Test this by checking if the parent 305 // directory would be included. 306 parentDir := filepath.Dir(filepath.Clean(item)) 307 308 parentFI, err := fs.Lstat(parentDir) 309 if err != nil { 310 debug.Log("item %v: error running lstat() on parent directory: %v", item, err) 311 // if in doubt, reject 312 return true 313 } 314 315 parentDeviceID, err := fs.DeviceID(parentFI) 316 if err != nil { 317 debug.Log("item %v: getting device ID of parent directory: %v", item, err) 318 // if in doubt, reject 319 return true 320 } 321 322 parentAllowed, err := deviceMap.IsAllowed(parentDir, parentDeviceID) 323 if err != nil { 324 debug.Log("item %v: error checking parent directory: %v", item, err) 325 // if in doubt, reject 326 return true 327 } 328 329 if parentAllowed { 330 // we found a mount point, so accept the directory 331 return false 332 } 333 334 // reject everything else 335 return true 336 }, nil 337} 338 339// rejectResticCache returns a RejectByNameFunc that rejects the restic cache 340// directory (if set). 341func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) { 342 if repo.Cache == nil { 343 return func(string) bool { 344 return false 345 }, nil 346 } 347 cacheBase := repo.Cache.BaseDir() 348 349 if cacheBase == "" { 350 return nil, errors.New("cacheBase is empty string") 351 } 352 353 return func(item string) bool { 354 if fs.HasPathPrefix(cacheBase, item) { 355 debug.Log("rejecting restic cache directory %v", item) 356 return true 357 } 358 359 return false 360 }, nil 361} 362 363func rejectBySize(maxSizeStr string) (RejectFunc, error) { 364 maxSize, err := parseSizeStr(maxSizeStr) 365 if err != nil { 366 return nil, err 367 } 368 369 return func(item string, fi os.FileInfo) bool { 370 // directory will be ignored 371 if fi.IsDir() { 372 return false 373 } 374 375 filesize := fi.Size() 376 if filesize > maxSize { 377 debug.Log("file %s is oversize: %d", item, filesize) 378 return true 379 } 380 381 return false 382 }, nil 383} 384 385func parseSizeStr(sizeStr string) (int64, error) { 386 if sizeStr == "" { 387 return 0, errors.New("expected size, got empty string") 388 } 389 390 numStr := sizeStr[:len(sizeStr)-1] 391 var unit int64 = 1 392 393 switch sizeStr[len(sizeStr)-1] { 394 case 'b', 'B': 395 // use initialized values, do nothing here 396 case 'k', 'K': 397 unit = 1024 398 case 'm', 'M': 399 unit = 1024 * 1024 400 case 'g', 'G': 401 unit = 1024 * 1024 * 1024 402 case 't', 'T': 403 unit = 1024 * 1024 * 1024 * 1024 404 default: 405 numStr = sizeStr 406 } 407 value, err := strconv.ParseInt(numStr, 10, 64) 408 if err != nil { 409 return 0, err 410 } 411 return value * unit, nil 412} 413