1// Copyright 2019 The Hugo Authors. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package helpers 15 16import ( 17 "errors" 18 "fmt" 19 "io" 20 "os" 21 "path/filepath" 22 "regexp" 23 "sort" 24 "strings" 25 "unicode" 26 27 "github.com/gohugoio/hugo/common/text" 28 29 "github.com/gohugoio/hugo/config" 30 31 "github.com/gohugoio/hugo/hugofs" 32 33 "github.com/gohugoio/hugo/common/hugio" 34 _errors "github.com/pkg/errors" 35 "github.com/spf13/afero" 36) 37 38// ErrThemeUndefined is returned when a theme has not be defined by the user. 39var ErrThemeUndefined = errors.New("no theme set") 40 41// MakePath takes a string with any characters and replace it 42// so the string could be used in a path. 43// It does so by creating a Unicode-sanitized string, with the spaces replaced, 44// whilst preserving the original casing of the string. 45// E.g. Social Media -> Social-Media 46func (p *PathSpec) MakePath(s string) string { 47 return p.UnicodeSanitize(s) 48} 49 50// MakePathsSanitized applies MakePathSanitized on every item in the slice 51func (p *PathSpec) MakePathsSanitized(paths []string) { 52 for i, path := range paths { 53 paths[i] = p.MakePathSanitized(path) 54 } 55} 56 57// MakePathSanitized creates a Unicode-sanitized string, with the spaces replaced 58func (p *PathSpec) MakePathSanitized(s string) string { 59 if p.DisablePathToLower { 60 return p.MakePath(s) 61 } 62 return strings.ToLower(p.MakePath(s)) 63} 64 65// ToSlashTrimLeading is just a filepath.ToSlaas with an added / prefix trimmer. 66func ToSlashTrimLeading(s string) string { 67 return strings.TrimPrefix(filepath.ToSlash(s), "/") 68} 69 70// MakeTitle converts the path given to a suitable title, trimming whitespace 71// and replacing hyphens with whitespace. 72func MakeTitle(inpath string) string { 73 return strings.Replace(strings.TrimSpace(inpath), "-", " ", -1) 74} 75 76// From https://golang.org/src/net/url/url.go 77func ishex(c rune) bool { 78 switch { 79 case '0' <= c && c <= '9': 80 return true 81 case 'a' <= c && c <= 'f': 82 return true 83 case 'A' <= c && c <= 'F': 84 return true 85 } 86 return false 87} 88 89// UnicodeSanitize sanitizes string to be used in Hugo URL's, allowing only 90// a predefined set of special Unicode characters. 91// If RemovePathAccents configuration flag is enabled, Unicode accents 92// are also removed. 93// Spaces will be replaced with a single hyphen, and sequential hyphens will be reduced to one. 94func (p *PathSpec) UnicodeSanitize(s string) string { 95 if p.RemovePathAccents { 96 s = text.RemoveAccentsString(s) 97 } 98 99 source := []rune(s) 100 target := make([]rune, 0, len(source)) 101 var prependHyphen bool 102 103 for i, r := range source { 104 isAllowed := r == '.' || r == '/' || r == '\\' || r == '_' || r == '#' || r == '+' || r == '~' 105 isAllowed = isAllowed || unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r) 106 isAllowed = isAllowed || (r == '%' && i+2 < len(source) && ishex(source[i+1]) && ishex(source[i+2])) 107 108 if isAllowed { 109 if prependHyphen { 110 target = append(target, '-') 111 prependHyphen = false 112 } 113 target = append(target, r) 114 } else if len(target) > 0 && (r == '-' || unicode.IsSpace(r)) { 115 prependHyphen = true 116 } 117 } 118 119 return string(target) 120} 121 122func makePathRelative(inPath string, possibleDirectories ...string) (string, error) { 123 for _, currentPath := range possibleDirectories { 124 if strings.HasPrefix(inPath, currentPath) { 125 return strings.TrimPrefix(inPath, currentPath), nil 126 } 127 } 128 return inPath, errors.New("can't extract relative path, unknown prefix") 129} 130 131// Should be good enough for Hugo. 132var isFileRe = regexp.MustCompile(`.*\..{1,6}$`) 133 134// GetDottedRelativePath expects a relative path starting after the content directory. 135// It returns a relative path with dots ("..") navigating up the path structure. 136func GetDottedRelativePath(inPath string) string { 137 inPath = filepath.Clean(filepath.FromSlash(inPath)) 138 139 if inPath == "." { 140 return "./" 141 } 142 143 if !isFileRe.MatchString(inPath) && !strings.HasSuffix(inPath, FilePathSeparator) { 144 inPath += FilePathSeparator 145 } 146 147 if !strings.HasPrefix(inPath, FilePathSeparator) { 148 inPath = FilePathSeparator + inPath 149 } 150 151 dir, _ := filepath.Split(inPath) 152 153 sectionCount := strings.Count(dir, FilePathSeparator) 154 155 if sectionCount == 0 || dir == FilePathSeparator { 156 return "./" 157 } 158 159 var dottedPath string 160 161 for i := 1; i < sectionCount; i++ { 162 dottedPath += "../" 163 } 164 165 return dottedPath 166} 167 168type NamedSlice struct { 169 Name string 170 Slice []string 171} 172 173func (n NamedSlice) String() string { 174 if len(n.Slice) == 0 { 175 return n.Name 176 } 177 return fmt.Sprintf("%s%s{%s}", n.Name, FilePathSeparator, strings.Join(n.Slice, ",")) 178} 179 180func ExtractAndGroupRootPaths(paths []string) []NamedSlice { 181 if len(paths) == 0 { 182 return nil 183 } 184 185 pathsCopy := make([]string, len(paths)) 186 hadSlashPrefix := strings.HasPrefix(paths[0], FilePathSeparator) 187 188 for i, p := range paths { 189 pathsCopy[i] = strings.Trim(filepath.ToSlash(p), "/") 190 } 191 192 sort.Strings(pathsCopy) 193 194 pathsParts := make([][]string, len(pathsCopy)) 195 196 for i, p := range pathsCopy { 197 pathsParts[i] = strings.Split(p, "/") 198 } 199 200 var groups [][]string 201 202 for i, p1 := range pathsParts { 203 c1 := -1 204 205 for j, p2 := range pathsParts { 206 if i == j { 207 continue 208 } 209 210 c2 := -1 211 212 for i, v := range p1 { 213 if i >= len(p2) { 214 break 215 } 216 if v != p2[i] { 217 break 218 } 219 220 c2 = i 221 } 222 223 if c1 == -1 || (c2 != -1 && c2 < c1) { 224 c1 = c2 225 } 226 } 227 228 if c1 != -1 { 229 groups = append(groups, p1[:c1+1]) 230 } else { 231 groups = append(groups, p1) 232 } 233 } 234 235 groupsStr := make([]string, len(groups)) 236 for i, g := range groups { 237 groupsStr[i] = strings.Join(g, "/") 238 } 239 240 groupsStr = UniqueStringsSorted(groupsStr) 241 242 var result []NamedSlice 243 244 for _, g := range groupsStr { 245 name := filepath.FromSlash(g) 246 if hadSlashPrefix { 247 name = FilePathSeparator + name 248 } 249 ns := NamedSlice{Name: name} 250 for _, p := range pathsCopy { 251 if !strings.HasPrefix(p, g) { 252 continue 253 } 254 255 p = strings.TrimPrefix(p, g) 256 if p != "" { 257 ns.Slice = append(ns.Slice, p) 258 } 259 } 260 261 ns.Slice = UniqueStrings(ExtractRootPaths(ns.Slice)) 262 263 result = append(result, ns) 264 } 265 266 return result 267} 268 269// ExtractRootPaths extracts the root paths from the supplied list of paths. 270// The resulting root path will not contain any file separators, but there 271// may be duplicates. 272// So "/content/section/" becomes "content" 273func ExtractRootPaths(paths []string) []string { 274 r := make([]string, len(paths)) 275 for i, p := range paths { 276 root := filepath.ToSlash(p) 277 sections := strings.Split(root, "/") 278 for _, section := range sections { 279 if section != "" { 280 root = section 281 break 282 } 283 } 284 r[i] = root 285 } 286 return r 287} 288 289// FindCWD returns the current working directory from where the Hugo 290// executable is run. 291func FindCWD() (string, error) { 292 serverFile, err := filepath.Abs(os.Args[0]) 293 if err != nil { 294 return "", fmt.Errorf("can't get absolute path for executable: %v", err) 295 } 296 297 path := filepath.Dir(serverFile) 298 realFile, err := filepath.EvalSymlinks(serverFile) 299 if err != nil { 300 if _, err = os.Stat(serverFile + ".exe"); err == nil { 301 realFile = filepath.Clean(serverFile + ".exe") 302 } 303 } 304 305 if err == nil && realFile != serverFile { 306 path = filepath.Dir(realFile) 307 } 308 309 return path, nil 310} 311 312// SymbolicWalk is like filepath.Walk, but it follows symbolic links. 313func SymbolicWalk(fs afero.Fs, root string, walker hugofs.WalkFunc) error { 314 if _, isOs := fs.(*afero.OsFs); isOs { 315 // Mainly to track symlinks. 316 fs = hugofs.NewBaseFileDecorator(fs) 317 } 318 319 w := hugofs.NewWalkway(hugofs.WalkwayConfig{ 320 Fs: fs, 321 Root: root, 322 WalkFn: walker, 323 }) 324 325 return w.Walk() 326} 327 328// LstatIfPossible can be used to call Lstat if possible, else Stat. 329func LstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) { 330 if lstater, ok := fs.(afero.Lstater); ok { 331 fi, _, err := lstater.LstatIfPossible(path) 332 return fi, err 333 } 334 335 return fs.Stat(path) 336} 337 338// SafeWriteToDisk is the same as WriteToDisk 339// but it also checks to see if file/directory already exists. 340func SafeWriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) { 341 return afero.SafeWriteReader(fs, inpath, r) 342} 343 344// WriteToDisk writes content to disk. 345func WriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) { 346 return afero.WriteReader(fs, inpath, r) 347} 348 349// OpenFilesForWriting opens all the given filenames for writing. 350func OpenFilesForWriting(fs afero.Fs, filenames ...string) (io.WriteCloser, error) { 351 var writeClosers []io.WriteCloser 352 for _, filename := range filenames { 353 f, err := OpenFileForWriting(fs, filename) 354 if err != nil { 355 for _, wc := range writeClosers { 356 wc.Close() 357 } 358 return nil, err 359 } 360 writeClosers = append(writeClosers, f) 361 } 362 363 return hugio.NewMultiWriteCloser(writeClosers...), nil 364} 365 366// OpenFileForWriting opens or creates the given file. If the target directory 367// does not exist, it gets created. 368func OpenFileForWriting(fs afero.Fs, filename string) (afero.File, error) { 369 filename = filepath.Clean(filename) 370 // Create will truncate if file already exists. 371 // os.Create will create any new files with mode 0666 (before umask). 372 f, err := fs.Create(filename) 373 if err != nil { 374 if !os.IsNotExist(err) { 375 return nil, err 376 } 377 if err = fs.MkdirAll(filepath.Dir(filename), 0777); err != nil { // before umask 378 return nil, err 379 } 380 f, err = fs.Create(filename) 381 } 382 383 return f, err 384} 385 386// GetCacheDir returns a cache dir from the given filesystem and config. 387// The dir will be created if it does not exist. 388func GetCacheDir(fs afero.Fs, cfg config.Provider) (string, error) { 389 cacheDir := getCacheDir(cfg) 390 if cacheDir != "" { 391 exists, err := DirExists(cacheDir, fs) 392 if err != nil { 393 return "", err 394 } 395 if !exists { 396 err := fs.MkdirAll(cacheDir, 0777) // Before umask 397 if err != nil { 398 return "", _errors.Wrap(err, "failed to create cache dir") 399 } 400 } 401 return cacheDir, nil 402 } 403 404 // Fall back to a cache in /tmp. 405 return GetTempDir("hugo_cache", fs), nil 406} 407 408func getCacheDir(cfg config.Provider) string { 409 // Always use the cacheDir config if set. 410 cacheDir := cfg.GetString("cacheDir") 411 if len(cacheDir) > 1 { 412 return addTrailingFileSeparator(cacheDir) 413 } 414 415 // See Issue #8714. 416 // Turns out that Cloudflare also sets NETLIFY=true in its build environment, 417 // but all of these 3 should not give any false positives. 418 if os.Getenv("NETLIFY") == "true" && os.Getenv("PULL_REQUEST") != "" && os.Getenv("DEPLOY_PRIME_URL") != "" { 419 // Netlify's cache behaviour is not documented, the currently best example 420 // is this project: 421 // https://github.com/philhawksworth/content-shards/blob/master/gulpfile.js 422 return "/opt/build/cache/hugo_cache/" 423 } 424 425 // This will fall back to an hugo_cache folder in the tmp dir, which should work fine for most CI 426 // providers. See this for a working CircleCI setup: 427 // https://github.com/bep/hugo-sass-test/blob/6c3960a8f4b90e8938228688bc49bdcdd6b2d99e/.circleci/config.yml 428 // If not, they can set the HUGO_CACHEDIR environment variable or cacheDir config key. 429 return "" 430} 431 432func addTrailingFileSeparator(s string) string { 433 if !strings.HasSuffix(s, FilePathSeparator) { 434 s = s + FilePathSeparator 435 } 436 return s 437} 438 439// GetTempDir returns a temporary directory with the given sub path. 440func GetTempDir(subPath string, fs afero.Fs) string { 441 return afero.GetTempDir(fs, subPath) 442} 443 444// DirExists checks if a path exists and is a directory. 445func DirExists(path string, fs afero.Fs) (bool, error) { 446 return afero.DirExists(fs, path) 447} 448 449// IsDir checks if a given path is a directory. 450func IsDir(path string, fs afero.Fs) (bool, error) { 451 return afero.IsDir(fs, path) 452} 453 454// IsEmpty checks if a given path is empty. 455func IsEmpty(path string, fs afero.Fs) (bool, error) { 456 return afero.IsEmpty(fs, path) 457} 458 459// FileContains checks if a file contains a specified string. 460func FileContains(filename string, subslice []byte, fs afero.Fs) (bool, error) { 461 return afero.FileContainsBytes(fs, filename, subslice) 462} 463 464// FileContainsAny checks if a file contains any of the specified strings. 465func FileContainsAny(filename string, subslices [][]byte, fs afero.Fs) (bool, error) { 466 return afero.FileContainsAnyBytes(fs, filename, subslices) 467} 468 469// Exists checks if a file or directory exists. 470func Exists(path string, fs afero.Fs) (bool, error) { 471 return afero.Exists(fs, path) 472} 473 474// AddTrailingSlash adds a trailing Unix styled slash (/) if not already 475// there. 476func AddTrailingSlash(path string) string { 477 if !strings.HasSuffix(path, "/") { 478 path += "/" 479 } 480 return path 481} 482