1package seelog 2 3import ( 4 "fmt" 5 "io" 6 "os" 7 "path/filepath" 8 "sync" 9) 10 11// File and directory permitions. 12const ( 13 defaultFilePermissions = 0666 14 defaultDirectoryPermissions = 0767 15) 16 17const ( 18 // Max number of directories can be read asynchronously. 19 maxDirNumberReadAsync = 1000 20) 21 22type cannotOpenFileError struct { 23 baseError 24} 25 26func newCannotOpenFileError(fname string) *cannotOpenFileError { 27 return &cannotOpenFileError{baseError{message: "Cannot open file: " + fname}} 28} 29 30type notDirectoryError struct { 31 baseError 32} 33 34func newNotDirectoryError(dname string) *notDirectoryError { 35 return ¬DirectoryError{baseError{message: dname + " is not directory"}} 36} 37 38// fileFilter is a filtering criteria function for '*os.File'. 39// Must return 'false' to set aside the given file. 40type fileFilter func(os.FileInfo, *os.File) bool 41 42// filePathFilter is a filtering creteria function for file path. 43// Must return 'false' to set aside the given file. 44type filePathFilter func(filePath string) bool 45 46// GetSubdirNames returns a list of directories found in 47// the given one with dirPath. 48func getSubdirNames(dirPath string) ([]string, error) { 49 fi, err := os.Stat(dirPath) 50 if err != nil { 51 return nil, err 52 } 53 if !fi.IsDir() { 54 return nil, newNotDirectoryError(dirPath) 55 } 56 dd, err := os.Open(dirPath) 57 // Cannot open file. 58 if err != nil { 59 if dd != nil { 60 dd.Close() 61 } 62 return nil, err 63 } 64 defer dd.Close() 65 // TODO: Improve performance by buffering reading. 66 allEntities, err := dd.Readdir(-1) 67 if err != nil { 68 return nil, err 69 } 70 subDirs := []string{} 71 for _, entity := range allEntities { 72 if entity.IsDir() { 73 subDirs = append(subDirs, entity.Name()) 74 } 75 } 76 return subDirs, nil 77} 78 79// getSubdirAbsPaths recursively visit all the subdirectories 80// starting from the given directory and returns absolute paths for them. 81func getAllSubdirAbsPaths(dirPath string) (res []string, err error) { 82 dps, err := getSubdirAbsPaths(dirPath) 83 if err != nil { 84 res = []string{} 85 return 86 } 87 res = append(res, dps...) 88 for _, dp := range dps { 89 sdps, err := getAllSubdirAbsPaths(dp) 90 if err != nil { 91 return []string{}, err 92 } 93 res = append(res, sdps...) 94 } 95 return 96} 97 98// getSubdirAbsPaths supplies absolute paths for all subdirectiries in a given directory. 99// Input: (I1) dirPath - absolute path of a directory in question. 100// Out: (O1) - slice of subdir asbolute paths; (O2) - error of the operation. 101// Remark: If error (O2) is non-nil then (O1) is nil and vice versa. 102func getSubdirAbsPaths(dirPath string) ([]string, error) { 103 sdns, err := getSubdirNames(dirPath) 104 if err != nil { 105 return nil, err 106 } 107 rsdns := []string{} 108 for _, sdn := range sdns { 109 rsdns = append(rsdns, filepath.Join(dirPath, sdn)) 110 } 111 return rsdns, nil 112} 113 114// getOpenFilesInDir supplies a slice of os.File pointers to files located in the directory. 115// Remark: Ignores files for which fileFilter returns false 116func getOpenFilesInDir(dirPath string, fFilter fileFilter) ([]*os.File, error) { 117 dfi, err := os.Open(dirPath) 118 if err != nil { 119 return nil, newCannotOpenFileError("Cannot open directory " + dirPath) 120 } 121 defer dfi.Close() 122 // Size of read buffer (i.e. chunk of items read at a time). 123 rbs := 64 124 resFiles := []*os.File{} 125L: 126 for { 127 // Read directory entities by reasonable chuncks 128 // to prevent overflows on big number of files. 129 fis, e := dfi.Readdir(rbs) 130 switch e { 131 // It's OK. 132 case nil: 133 // Do nothing, just continue cycle. 134 case io.EOF: 135 break L 136 // Something went wrong. 137 default: 138 return nil, e 139 } 140 // THINK: Maybe, use async running. 141 for _, fi := range fis { 142 // NB: On Linux this could be a problem as 143 // there are lots of file types available. 144 if !fi.IsDir() { 145 f, e := os.Open(filepath.Join(dirPath, fi.Name())) 146 if e != nil { 147 if f != nil { 148 f.Close() 149 } 150 // THINK: Add nil as indicator that a problem occurred. 151 resFiles = append(resFiles, nil) 152 continue 153 } 154 // Check filter condition. 155 if fFilter != nil && !fFilter(fi, f) { 156 continue 157 } 158 resFiles = append(resFiles, f) 159 } 160 } 161 } 162 return resFiles, nil 163} 164 165func isRegular(m os.FileMode) bool { 166 return m&os.ModeType == 0 167} 168 169// getDirFilePaths return full paths of the files located in the directory. 170// Remark: Ignores files for which fileFilter returns false. 171func getDirFilePaths(dirPath string, fpFilter filePathFilter, pathIsName bool) ([]string, error) { 172 dfi, err := os.Open(dirPath) 173 if err != nil { 174 return nil, newCannotOpenFileError("Cannot open directory " + dirPath) 175 } 176 defer dfi.Close() 177 178 var absDirPath string 179 if !filepath.IsAbs(dirPath) { 180 absDirPath, err = filepath.Abs(dirPath) 181 if err != nil { 182 return nil, fmt.Errorf("cannot get absolute path of directory: %s", err.Error()) 183 } 184 } else { 185 absDirPath = dirPath 186 } 187 188 // TODO: check if dirPath is really directory. 189 // Size of read buffer (i.e. chunk of items read at a time). 190 rbs := 2 << 5 191 filePaths := []string{} 192 193 var fp string 194L: 195 for { 196 // Read directory entities by reasonable chuncks 197 // to prevent overflows on big number of files. 198 fis, e := dfi.Readdir(rbs) 199 switch e { 200 // It's OK. 201 case nil: 202 // Do nothing, just continue cycle. 203 case io.EOF: 204 break L 205 // Indicate that something went wrong. 206 default: 207 return nil, e 208 } 209 // THINK: Maybe, use async running. 210 for _, fi := range fis { 211 // NB: Should work on every Windows and non-Windows OS. 212 if isRegular(fi.Mode()) { 213 if pathIsName { 214 fp = fi.Name() 215 } else { 216 // Build full path of a file. 217 fp = filepath.Join(absDirPath, fi.Name()) 218 } 219 // Check filter condition. 220 if fpFilter != nil && !fpFilter(fp) { 221 continue 222 } 223 filePaths = append(filePaths, fp) 224 } 225 } 226 } 227 return filePaths, nil 228} 229 230// getOpenFilesByDirectoryAsync runs async reading directories 'dirPaths' and inserts pairs 231// in map 'filesInDirMap': Key - directory name, value - *os.File slice. 232func getOpenFilesByDirectoryAsync( 233 dirPaths []string, 234 fFilter fileFilter, 235 filesInDirMap map[string][]*os.File, 236) error { 237 n := len(dirPaths) 238 if n > maxDirNumberReadAsync { 239 return fmt.Errorf("number of input directories to be read exceeded max value %d", maxDirNumberReadAsync) 240 } 241 type filesInDirResult struct { 242 DirName string 243 Files []*os.File 244 Error error 245 } 246 dirFilesChan := make(chan *filesInDirResult, n) 247 var wg sync.WaitGroup 248 // Register n goroutines which are going to do work. 249 wg.Add(n) 250 for i := 0; i < n; i++ { 251 // Launch asynchronously the piece of work. 252 go func(dirPath string) { 253 fs, e := getOpenFilesInDir(dirPath, fFilter) 254 dirFilesChan <- &filesInDirResult{filepath.Base(dirPath), fs, e} 255 // Mark the current goroutine as finished (work is done). 256 wg.Done() 257 }(dirPaths[i]) 258 } 259 // Wait for all goroutines to finish their work. 260 wg.Wait() 261 // Close the error channel to let for-range clause 262 // get all the buffered values without blocking and quit in the end. 263 close(dirFilesChan) 264 for fidr := range dirFilesChan { 265 if fidr.Error == nil { 266 // THINK: What will happen if the key is already present? 267 filesInDirMap[fidr.DirName] = fidr.Files 268 } else { 269 return fidr.Error 270 } 271 } 272 return nil 273} 274 275// fileExists return flag whether a given file exists 276// and operation error if an unclassified failure occurs. 277func fileExists(path string) (bool, error) { 278 _, err := os.Stat(path) 279 if err != nil { 280 if os.IsNotExist(err) { 281 return false, nil 282 } 283 return false, err 284 } 285 return true, nil 286} 287 288// createDirectory makes directory with a given name 289// making all parent directories if necessary. 290func createDirectory(dirPath string) error { 291 var dPath string 292 var err error 293 if !filepath.IsAbs(dirPath) { 294 dPath, err = filepath.Abs(dirPath) 295 if err != nil { 296 return err 297 } 298 } else { 299 dPath = dirPath 300 } 301 exists, err := fileExists(dPath) 302 if err != nil { 303 return err 304 } 305 if exists { 306 return nil 307 } 308 return os.MkdirAll(dPath, os.ModeDir) 309} 310 311// tryRemoveFile gives a try removing the file 312// only ignoring an error when the file does not exist. 313func tryRemoveFile(filePath string) (err error) { 314 err = os.Remove(filePath) 315 if os.IsNotExist(err) { 316 err = nil 317 return 318 } 319 return 320} 321