1// Copyright 2019 The Hugo Authors. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package helpers 15 16import ( 17 "bytes" 18 "crypto/md5" 19 "encoding/hex" 20 "fmt" 21 "io" 22 "net" 23 "os" 24 "path/filepath" 25 "sort" 26 "strconv" 27 "strings" 28 "sync" 29 "unicode" 30 "unicode/utf8" 31 32 "github.com/gohugoio/hugo/common/loggers" 33 34 "github.com/mitchellh/hashstructure" 35 36 "github.com/gohugoio/hugo/hugofs" 37 38 "github.com/gohugoio/hugo/common/hugo" 39 40 "github.com/spf13/afero" 41 42 "github.com/jdkato/prose/transform" 43 44 bp "github.com/gohugoio/hugo/bufferpool" 45 "github.com/spf13/pflag" 46) 47 48// FilePathSeparator as defined by os.Separator. 49const FilePathSeparator = string(filepath.Separator) 50 51// FindAvailablePort returns an available and valid TCP port. 52func FindAvailablePort() (*net.TCPAddr, error) { 53 l, err := net.Listen("tcp", ":0") 54 if err == nil { 55 defer l.Close() 56 addr := l.Addr() 57 if a, ok := addr.(*net.TCPAddr); ok { 58 return a, nil 59 } 60 return nil, fmt.Errorf("unable to obtain a valid tcp port: %v", addr) 61 } 62 return nil, err 63} 64 65// InStringArray checks if a string is an element of a slice of strings 66// and returns a boolean value. 67func InStringArray(arr []string, el string) bool { 68 for _, v := range arr { 69 if v == el { 70 return true 71 } 72 } 73 return false 74} 75 76// FirstUpper returns a string with the first character as upper case. 77func FirstUpper(s string) string { 78 if s == "" { 79 return "" 80 } 81 r, n := utf8.DecodeRuneInString(s) 82 return string(unicode.ToUpper(r)) + s[n:] 83} 84 85// UniqueStrings returns a new slice with any duplicates removed. 86func UniqueStrings(s []string) []string { 87 unique := make([]string, 0, len(s)) 88 for i, val := range s { 89 var seen bool 90 for j := 0; j < i; j++ { 91 if s[j] == val { 92 seen = true 93 break 94 } 95 } 96 if !seen { 97 unique = append(unique, val) 98 } 99 } 100 return unique 101} 102 103// UniqueStringsReuse returns a slice with any duplicates removed. 104// It will modify the input slice. 105func UniqueStringsReuse(s []string) []string { 106 result := s[:0] 107 for i, val := range s { 108 var seen bool 109 110 for j := 0; j < i; j++ { 111 if s[j] == val { 112 seen = true 113 break 114 } 115 } 116 117 if !seen { 118 result = append(result, val) 119 } 120 } 121 return result 122} 123 124// UniqueStringsReuse returns a sorted slice with any duplicates removed. 125// It will modify the input slice. 126func UniqueStringsSorted(s []string) []string { 127 if len(s) == 0 { 128 return nil 129 } 130 ss := sort.StringSlice(s) 131 ss.Sort() 132 i := 0 133 for j := 1; j < len(s); j++ { 134 if !ss.Less(i, j) { 135 continue 136 } 137 i++ 138 s[i] = s[j] 139 } 140 141 return s[:i+1] 142} 143 144// ReaderToBytes takes an io.Reader argument, reads from it 145// and returns bytes. 146func ReaderToBytes(lines io.Reader) []byte { 147 if lines == nil { 148 return []byte{} 149 } 150 b := bp.GetBuffer() 151 defer bp.PutBuffer(b) 152 153 b.ReadFrom(lines) 154 155 bc := make([]byte, b.Len()) 156 copy(bc, b.Bytes()) 157 return bc 158} 159 160// ReaderToString is the same as ReaderToBytes, but returns a string. 161func ReaderToString(lines io.Reader) string { 162 if lines == nil { 163 return "" 164 } 165 b := bp.GetBuffer() 166 defer bp.PutBuffer(b) 167 b.ReadFrom(lines) 168 return b.String() 169} 170 171// ReaderContains reports whether subslice is within r. 172func ReaderContains(r io.Reader, subslice []byte) bool { 173 if r == nil || len(subslice) == 0 { 174 return false 175 } 176 177 bufflen := len(subslice) * 4 178 halflen := bufflen / 2 179 buff := make([]byte, bufflen) 180 var err error 181 var n, i int 182 183 for { 184 i++ 185 if i == 1 { 186 n, err = io.ReadAtLeast(r, buff[:halflen], halflen) 187 } else { 188 if i != 2 { 189 // shift left to catch overlapping matches 190 copy(buff[:], buff[halflen:]) 191 } 192 n, err = io.ReadAtLeast(r, buff[halflen:], halflen) 193 } 194 195 if n > 0 && bytes.Contains(buff, subslice) { 196 return true 197 } 198 199 if err != nil { 200 break 201 } 202 } 203 return false 204} 205 206// GetTitleFunc returns a func that can be used to transform a string to 207// title case. 208// 209// The supported styles are 210// 211// - "Go" (strings.Title) 212// - "AP" (see https://www.apstylebook.com/) 213// - "Chicago" (see http://www.chicagomanualofstyle.org/home.html) 214// 215// If an unknown or empty style is provided, AP style is what you get. 216func GetTitleFunc(style string) func(s string) string { 217 switch strings.ToLower(style) { 218 case "go": 219 return strings.Title 220 case "chicago": 221 tc := transform.NewTitleConverter(transform.ChicagoStyle) 222 return tc.Title 223 default: 224 tc := transform.NewTitleConverter(transform.APStyle) 225 return tc.Title 226 } 227} 228 229// HasStringsPrefix tests whether the string slice s begins with prefix slice s. 230func HasStringsPrefix(s, prefix []string) bool { 231 return len(s) >= len(prefix) && compareStringSlices(s[0:len(prefix)], prefix) 232} 233 234// HasStringsSuffix tests whether the string slice s ends with suffix slice s. 235func HasStringsSuffix(s, suffix []string) bool { 236 return len(s) >= len(suffix) && compareStringSlices(s[len(s)-len(suffix):], suffix) 237} 238 239func compareStringSlices(a, b []string) bool { 240 if a == nil && b == nil { 241 return true 242 } 243 244 if a == nil || b == nil { 245 return false 246 } 247 248 if len(a) != len(b) { 249 return false 250 } 251 252 for i := range a { 253 if a[i] != b[i] { 254 return false 255 } 256 } 257 258 return true 259} 260 261// LogPrinter is the common interface of the JWWs loggers. 262type LogPrinter interface { 263 // Println is the only common method that works in all of JWWs loggers. 264 Println(a ...interface{}) 265} 266 267// DistinctLogger ignores duplicate log statements. 268type DistinctLogger struct { 269 loggers.Logger 270 sync.RWMutex 271 m map[string]bool 272} 273 274func (l *DistinctLogger) Reset() { 275 l.Lock() 276 defer l.Unlock() 277 278 l.m = make(map[string]bool) 279} 280 281// Println will log the string returned from fmt.Sprintln given the arguments, 282// but not if it has been logged before. 283func (l *DistinctLogger) Println(v ...interface{}) { 284 // fmt.Sprint doesn't add space between string arguments 285 logStatement := strings.TrimSpace(fmt.Sprintln(v...)) 286 l.printIfNotPrinted("println", logStatement, func() { 287 l.Logger.Println(logStatement) 288 }) 289} 290 291// Printf will log the string returned from fmt.Sprintf given the arguments, 292// but not if it has been logged before. 293func (l *DistinctLogger) Printf(format string, v ...interface{}) { 294 logStatement := fmt.Sprintf(format, v...) 295 l.printIfNotPrinted("printf", logStatement, func() { 296 l.Logger.Printf(format, v...) 297 }) 298} 299 300func (l *DistinctLogger) Debugf(format string, v ...interface{}) { 301 logStatement := fmt.Sprintf(format, v...) 302 l.printIfNotPrinted("debugf", logStatement, func() { 303 l.Logger.Debugf(format, v...) 304 }) 305} 306 307func (l *DistinctLogger) Debugln(v ...interface{}) { 308 logStatement := fmt.Sprint(v...) 309 l.printIfNotPrinted("debugln", logStatement, func() { 310 l.Logger.Debugln(v...) 311 }) 312} 313 314func (l *DistinctLogger) Infof(format string, v ...interface{}) { 315 logStatement := fmt.Sprintf(format, v...) 316 l.printIfNotPrinted("info", logStatement, func() { 317 l.Logger.Infof(format, v...) 318 }) 319} 320 321func (l *DistinctLogger) Infoln(v ...interface{}) { 322 logStatement := fmt.Sprint(v...) 323 l.printIfNotPrinted("infoln", logStatement, func() { 324 l.Logger.Infoln(v...) 325 }) 326} 327 328func (l *DistinctLogger) Warnf(format string, v ...interface{}) { 329 logStatement := fmt.Sprintf(format, v...) 330 l.printIfNotPrinted("warnf", logStatement, func() { 331 l.Logger.Warnf(format, v...) 332 }) 333} 334func (l *DistinctLogger) Warnln(v ...interface{}) { 335 logStatement := fmt.Sprint(v...) 336 l.printIfNotPrinted("warnln", logStatement, func() { 337 l.Logger.Warnln(v...) 338 }) 339} 340func (l *DistinctLogger) Errorf(format string, v ...interface{}) { 341 logStatement := fmt.Sprint(v...) 342 l.printIfNotPrinted("errorf", logStatement, func() { 343 l.Logger.Errorf(format, v...) 344 }) 345} 346 347func (l *DistinctLogger) Errorln(v ...interface{}) { 348 logStatement := fmt.Sprint(v...) 349 l.printIfNotPrinted("errorln", logStatement, func() { 350 l.Logger.Errorln(v...) 351 }) 352} 353 354func (l *DistinctLogger) hasPrinted(key string) bool { 355 l.RLock() 356 defer l.RUnlock() 357 _, found := l.m[key] 358 return found 359} 360 361func (l *DistinctLogger) printIfNotPrinted(level, logStatement string, print func()) { 362 key := level + logStatement 363 if l.hasPrinted(key) { 364 return 365 } 366 l.Lock() 367 print() 368 l.m[key] = true 369 l.Unlock() 370} 371 372// NewDistinctErrorLogger creates a new DistinctLogger that logs ERRORs 373func NewDistinctErrorLogger() loggers.Logger { 374 return &DistinctLogger{m: make(map[string]bool), Logger: loggers.NewErrorLogger()} 375} 376 377// NewDistinctLogger creates a new DistinctLogger that logs to the provided logger. 378func NewDistinctLogger(logger loggers.Logger) loggers.Logger { 379 return &DistinctLogger{m: make(map[string]bool), Logger: logger} 380} 381 382// NewDistinctWarnLogger creates a new DistinctLogger that logs WARNs 383func NewDistinctWarnLogger() loggers.Logger { 384 return &DistinctLogger{m: make(map[string]bool), Logger: loggers.NewWarningLogger()} 385} 386 387var ( 388 // DistinctErrorLog can be used to avoid spamming the logs with errors. 389 DistinctErrorLog = NewDistinctErrorLogger() 390 391 // DistinctWarnLog can be used to avoid spamming the logs with warnings. 392 DistinctWarnLog = NewDistinctWarnLogger() 393) 394 395// InitLoggers resets the global distinct loggers. 396func InitLoggers() { 397 DistinctErrorLog.Reset() 398 DistinctWarnLog.Reset() 399 400} 401 402// Deprecated informs about a deprecation, but only once for a given set of arguments' values. 403// If the err flag is enabled, it logs as an ERROR (will exit with -1) and the text will 404// point at the next Hugo release. 405// The idea is two remove an item in two Hugo releases to give users and theme authors 406// plenty of time to fix their templates. 407func Deprecated(item, alternative string, err bool) { 408 if err { 409 DistinctErrorLog.Errorf("%s is deprecated and will be removed in Hugo %s. %s", item, hugo.CurrentVersion.Next().ReleaseVersion(), alternative) 410 } else { 411 DistinctWarnLog.Warnf("%s is deprecated and will be removed in a future release. %s", item, alternative) 412 } 413} 414 415// SliceToLower goes through the source slice and lowers all values. 416func SliceToLower(s []string) []string { 417 if s == nil { 418 return nil 419 } 420 421 l := make([]string, len(s)) 422 for i, v := range s { 423 l[i] = strings.ToLower(v) 424 } 425 426 return l 427} 428 429// MD5String takes a string and returns its MD5 hash. 430func MD5String(f string) string { 431 h := md5.New() 432 h.Write([]byte(f)) 433 return hex.EncodeToString(h.Sum([]byte{})) 434} 435 436// MD5FromFileFast creates a MD5 hash from the given file. It only reads parts of 437// the file for speed, so don't use it if the files are very subtly different. 438// It will not close the file. 439func MD5FromFileFast(r io.ReadSeeker) (string, error) { 440 const ( 441 // Do not change once set in stone! 442 maxChunks = 8 443 peekSize = 64 444 seek = 2048 445 ) 446 447 h := md5.New() 448 buff := make([]byte, peekSize) 449 450 for i := 0; i < maxChunks; i++ { 451 if i > 0 { 452 _, err := r.Seek(seek, 0) 453 if err != nil { 454 if err == io.EOF { 455 break 456 } 457 return "", err 458 } 459 } 460 461 _, err := io.ReadAtLeast(r, buff, peekSize) 462 if err != nil { 463 if err == io.EOF || err == io.ErrUnexpectedEOF { 464 h.Write(buff) 465 break 466 } 467 return "", err 468 } 469 h.Write(buff) 470 } 471 472 return hex.EncodeToString(h.Sum(nil)), nil 473} 474 475// MD5FromReader creates a MD5 hash from the given reader. 476func MD5FromReader(r io.Reader) (string, error) { 477 h := md5.New() 478 if _, err := io.Copy(h, r); err != nil { 479 return "", nil 480 } 481 return hex.EncodeToString(h.Sum(nil)), nil 482} 483 484// IsWhitespace determines if the given rune is whitespace. 485func IsWhitespace(r rune) bool { 486 return r == ' ' || r == '\t' || r == '\n' || r == '\r' 487} 488 489// NormalizeHugoFlags facilitates transitions of Hugo command-line flags, 490// e.g. --baseUrl to --baseURL, --uglyUrls to --uglyURLs 491func NormalizeHugoFlags(f *pflag.FlagSet, name string) pflag.NormalizedName { 492 switch name { 493 case "baseUrl": 494 name = "baseURL" 495 case "uglyUrls": 496 name = "uglyURLs" 497 } 498 return pflag.NormalizedName(name) 499} 500 501// PrintFs prints the given filesystem to the given writer starting from the given path. 502// This is useful for debugging. 503func PrintFs(fs afero.Fs, path string, w io.Writer) { 504 if fs == nil { 505 return 506 } 507 508 afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { 509 var filename string 510 var meta interface{} 511 if fim, ok := info.(hugofs.FileMetaInfo); ok { 512 filename = fim.Meta().Filename 513 meta = fim.Meta() 514 } 515 fmt.Fprintf(w, " %q %q\t\t%v\n", path, filename, meta) 516 return nil 517 }) 518} 519 520// HashString returns a hash from the given elements. 521// It will panic if the hash cannot be calculated. 522func HashString(elements ...interface{}) string { 523 var o interface{} 524 if len(elements) == 1 { 525 o = elements[0] 526 } else { 527 o = elements 528 } 529 530 hash, err := hashstructure.Hash(o, nil) 531 if err != nil { 532 panic(err) 533 } 534 return strconv.FormatUint(hash, 10) 535} 536