1package scan 2 3import ( 4 "fmt" 5 "net/http" 6 "os" 7 "path/filepath" 8 "time" 9 10 "github.com/future-architect/vuls/cache" 11 "github.com/future-architect/vuls/config" 12 "github.com/future-architect/vuls/models" 13 "github.com/future-architect/vuls/report" 14 "github.com/future-architect/vuls/util" 15 "golang.org/x/xerrors" 16) 17 18const ( 19 scannedViaRemote = "remote" 20 scannedViaLocal = "local" 21 scannedViaPseudo = "pseudo" 22) 23 24var ( 25 errOSFamilyHeader = xerrors.New("X-Vuls-OS-Family header is required") 26 errOSReleaseHeader = xerrors.New("X-Vuls-OS-Release header is required") 27 errKernelVersionHeader = xerrors.New("X-Vuls-Kernel-Version header is required") 28 errServerNameHeader = xerrors.New("X-Vuls-Server-Name header is required") 29) 30 31var servers, errServers []osTypeInterface 32 33// Base Interface of redhat, debian, freebsd 34type osTypeInterface interface { 35 setServerInfo(config.ServerInfo) 36 getServerInfo() config.ServerInfo 37 setDistro(string, string) 38 getDistro() config.Distro 39 detectPlatform() 40 detectIPSs() 41 getPlatform() models.Platform 42 43 checkScanMode() error 44 checkDeps() error 45 checkIfSudoNoPasswd() error 46 47 preCure() error 48 postScan() error 49 scanWordPress() error 50 scanLibraries() error 51 scanPorts() error 52 scanPackages() error 53 convertToModel() models.ScanResult 54 55 parseInstalledPackages(string) (models.Packages, models.SrcPackages, error) 56 57 runningContainers() ([]config.Container, error) 58 exitedContainers() ([]config.Container, error) 59 allContainers() ([]config.Container, error) 60 61 getErrs() []error 62 setErrs([]error) 63} 64 65// osPackages is included by base struct 66type osPackages struct { 67 // installed packages 68 Packages models.Packages 69 70 // installed source packages (Debian based only) 71 SrcPackages models.SrcPackages 72 73 // unsecure packages 74 VulnInfos models.VulnInfos 75 76 // kernel information 77 Kernel models.Kernel 78} 79 80// Retry as it may stall on the first SSH connection 81// https://github.com/future-architect/vuls/pull/753 82func detectDebianWithRetry(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err error) { 83 type Response struct { 84 itsMe bool 85 deb osTypeInterface 86 err error 87 } 88 resChan := make(chan Response, 1) 89 go func(c config.ServerInfo) { 90 itsMe, osType, fatalErr := detectDebian(c) 91 resChan <- Response{itsMe, osType, fatalErr} 92 }(c) 93 94 timeout := time.After(time.Duration(3) * time.Second) 95 select { 96 case res := <-resChan: 97 return res.itsMe, res.deb, res.err 98 case <-timeout: 99 time.Sleep(100 * time.Millisecond) 100 return detectDebian(c) 101 } 102} 103 104func detectOS(c config.ServerInfo) (osType osTypeInterface) { 105 var itsMe bool 106 var fatalErr error 107 108 if itsMe, osType, _ = detectPseudo(c); itsMe { 109 util.Log.Debugf("Pseudo") 110 return 111 } 112 113 itsMe, osType, fatalErr = detectDebianWithRetry(c) 114 if fatalErr != nil { 115 osType.setErrs([]error{ 116 xerrors.Errorf("Failed to detect OS: %w", fatalErr)}) 117 return 118 } 119 120 if itsMe { 121 util.Log.Debugf("Debian like Linux. Host: %s:%s", c.Host, c.Port) 122 return 123 } 124 125 if itsMe, osType = detectRedhat(c); itsMe { 126 util.Log.Debugf("Redhat like Linux. Host: %s:%s", c.Host, c.Port) 127 return 128 } 129 130 if itsMe, osType = detectSUSE(c); itsMe { 131 util.Log.Debugf("SUSE Linux. Host: %s:%s", c.Host, c.Port) 132 return 133 } 134 135 if itsMe, osType = detectFreebsd(c); itsMe { 136 util.Log.Debugf("FreeBSD. Host: %s:%s", c.Host, c.Port) 137 return 138 } 139 140 if itsMe, osType = detectAlpine(c); itsMe { 141 util.Log.Debugf("Alpine. Host: %s:%s", c.Host, c.Port) 142 return 143 } 144 145 //TODO darwin https://github.com/mizzy/specinfra/blob/master/lib/specinfra/helper/detect_os/darwin.rb 146 osType.setErrs([]error{xerrors.New("Unknown OS Type")}) 147 return 148} 149 150// PrintSSHableServerNames print SSH-able servernames 151func PrintSSHableServerNames() bool { 152 if len(servers) == 0 { 153 util.Log.Error("No scannable servers") 154 return false 155 } 156 util.Log.Info("Scannable servers are below...") 157 for _, s := range servers { 158 if s.getServerInfo().IsContainer() { 159 fmt.Printf("%s@%s ", 160 s.getServerInfo().Container.Name, 161 s.getServerInfo().ServerName, 162 ) 163 } else { 164 fmt.Printf("%s ", s.getServerInfo().ServerName) 165 } 166 } 167 fmt.Printf("\n") 168 return true 169} 170 171// InitServers detect the kind of OS distribution of target servers 172func InitServers(timeoutSec int) error { 173 // use global servers, errServers when scan containers 174 servers, errServers = detectServerOSes(timeoutSec) 175 if len(servers) == 0 { 176 return xerrors.New("No scannable base servers") 177 } 178 179 // scan additional servers 180 var actives, inactives []osTypeInterface 181 oks, errs := detectContainerOSes(timeoutSec) 182 actives = append(actives, oks...) 183 inactives = append(inactives, errs...) 184 185 if config.Conf.ContainersOnly { 186 servers = actives 187 errServers = inactives 188 } else { 189 servers = append(servers, actives...) 190 errServers = append(errServers, inactives...) 191 } 192 193 if len(servers) == 0 { 194 return xerrors.New("No scannable servers") 195 } 196 return nil 197} 198 199func detectServerOSes(timeoutSec int) (servers, errServers []osTypeInterface) { 200 util.Log.Info("Detecting OS of servers... ") 201 osTypeChan := make(chan osTypeInterface, len(config.Conf.Servers)) 202 defer close(osTypeChan) 203 for _, s := range config.Conf.Servers { 204 go func(s config.ServerInfo) { 205 defer func() { 206 if p := recover(); p != nil { 207 util.Log.Debugf("Panic: %s on %s", p, s.ServerName) 208 } 209 }() 210 osTypeChan <- detectOS(s) 211 }(s) 212 } 213 214 timeout := time.After(time.Duration(timeoutSec) * time.Second) 215 for i := 0; i < len(config.Conf.Servers); i++ { 216 select { 217 case res := <-osTypeChan: 218 if 0 < len(res.getErrs()) { 219 errServers = append(errServers, res) 220 util.Log.Errorf("(%d/%d) Failed: %s, err: %+v", 221 i+1, len(config.Conf.Servers), 222 res.getServerInfo().ServerName, 223 res.getErrs()) 224 } else { 225 servers = append(servers, res) 226 util.Log.Infof("(%d/%d) Detected: %s: %s", 227 i+1, len(config.Conf.Servers), 228 res.getServerInfo().ServerName, 229 res.getDistro()) 230 } 231 case <-timeout: 232 msg := "Timed out while detecting servers" 233 util.Log.Error(msg) 234 for servername, sInfo := range config.Conf.Servers { 235 found := false 236 for _, o := range append(servers, errServers...) { 237 if servername == o.getServerInfo().ServerName { 238 found = true 239 break 240 } 241 } 242 if !found { 243 u := &unknown{} 244 u.setServerInfo(sInfo) 245 u.setErrs([]error{ 246 xerrors.New("Timed out"), 247 }) 248 errServers = append(errServers, u) 249 util.Log.Errorf("(%d/%d) Timed out: %s", 250 i+1, len(config.Conf.Servers), 251 servername) 252 i++ 253 } 254 } 255 } 256 } 257 return 258} 259 260func detectContainerOSes(timeoutSec int) (actives, inactives []osTypeInterface) { 261 util.Log.Info("Detecting OS of containers... ") 262 osTypesChan := make(chan []osTypeInterface, len(servers)) 263 defer close(osTypesChan) 264 for _, s := range servers { 265 go func(s osTypeInterface) { 266 defer func() { 267 if p := recover(); p != nil { 268 util.Log.Debugf("Panic: %s on %s", 269 p, s.getServerInfo().GetServerName()) 270 } 271 }() 272 osTypesChan <- detectContainerOSesOnServer(s) 273 }(s) 274 } 275 276 timeout := time.After(time.Duration(timeoutSec) * time.Second) 277 for i := 0; i < len(servers); i++ { 278 select { 279 case res := <-osTypesChan: 280 for _, osi := range res { 281 sinfo := osi.getServerInfo() 282 if 0 < len(osi.getErrs()) { 283 inactives = append(inactives, osi) 284 util.Log.Errorf("Failed: %s err: %+v", sinfo.ServerName, osi.getErrs()) 285 continue 286 } 287 actives = append(actives, osi) 288 util.Log.Infof("Detected: %s@%s: %s", 289 sinfo.Container.Name, sinfo.ServerName, osi.getDistro()) 290 } 291 case <-timeout: 292 msg := "Timed out while detecting containers" 293 util.Log.Error(msg) 294 for servername, sInfo := range config.Conf.Servers { 295 found := false 296 for _, o := range append(actives, inactives...) { 297 if servername == o.getServerInfo().ServerName { 298 found = true 299 break 300 } 301 } 302 if !found { 303 u := &unknown{} 304 u.setServerInfo(sInfo) 305 u.setErrs([]error{ 306 xerrors.New("Timed out"), 307 }) 308 util.Log.Errorf("Timed out: %s", servername) 309 } 310 } 311 } 312 } 313 return 314} 315 316func detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeInterface) { 317 containerHostInfo := containerHost.getServerInfo() 318 if len(containerHostInfo.ContainersIncluded) == 0 { 319 return 320 } 321 322 running, err := containerHost.runningContainers() 323 if err != nil { 324 containerHost.setErrs([]error{xerrors.Errorf( 325 "Failed to get running containers on %s. err: %w", 326 containerHost.getServerInfo().ServerName, err)}) 327 return append(oses, containerHost) 328 } 329 330 if containerHostInfo.ContainersIncluded[0] == "${running}" { 331 for _, containerInfo := range running { 332 found := false 333 for _, ex := range containerHost.getServerInfo().ContainersExcluded { 334 if containerInfo.Name == ex || containerInfo.ContainerID == ex { 335 found = true 336 } 337 } 338 if found { 339 continue 340 } 341 342 copied := containerHostInfo 343 copied.SetContainer(config.Container{ 344 ContainerID: containerInfo.ContainerID, 345 Name: containerInfo.Name, 346 Image: containerInfo.Image, 347 }) 348 os := detectOS(copied) 349 oses = append(oses, os) 350 } 351 return oses 352 } 353 354 exitedContainers, err := containerHost.exitedContainers() 355 if err != nil { 356 containerHost.setErrs([]error{xerrors.Errorf( 357 "Failed to get exited containers on %s. err: %w", 358 containerHost.getServerInfo().ServerName, err)}) 359 return append(oses, containerHost) 360 } 361 362 var exited, unknown []string 363 for _, container := range containerHostInfo.ContainersIncluded { 364 found := false 365 for _, c := range running { 366 if c.ContainerID == container || c.Name == container { 367 copied := containerHostInfo 368 copied.SetContainer(c) 369 os := detectOS(copied) 370 oses = append(oses, os) 371 found = true 372 break 373 } 374 } 375 376 if !found { 377 foundInExitedContainers := false 378 for _, c := range exitedContainers { 379 if c.ContainerID == container || c.Name == container { 380 exited = append(exited, container) 381 foundInExitedContainers = true 382 break 383 } 384 } 385 if !foundInExitedContainers { 386 unknown = append(unknown, container) 387 } 388 } 389 } 390 if 0 < len(exited) || 0 < len(unknown) { 391 containerHost.setErrs([]error{xerrors.Errorf( 392 "Some containers on %s are exited or unknown. exited: %s, unknown: %s", 393 containerHost.getServerInfo().ServerName, exited, unknown)}) 394 return append(oses, containerHost) 395 } 396 return oses 397} 398 399// CheckScanModes checks scan mode 400func CheckScanModes() error { 401 for _, s := range servers { 402 if err := s.checkScanMode(); err != nil { 403 return xerrors.Errorf("servers.%s.scanMode err: %w", 404 s.getServerInfo().GetServerName(), err) 405 } 406 } 407 return nil 408} 409 410// CheckDependencies checks dependencies are installed on target servers. 411func CheckDependencies(timeoutSec int) { 412 parallelExec(func(o osTypeInterface) error { 413 return o.checkDeps() 414 }, timeoutSec) 415 return 416} 417 418// CheckIfSudoNoPasswd checks whether vuls can sudo with nopassword via SSH 419func CheckIfSudoNoPasswd(timeoutSec int) { 420 parallelExec(func(o osTypeInterface) error { 421 return o.checkIfSudoNoPasswd() 422 }, timeoutSec) 423 return 424} 425 426// DetectPlatforms detects the platform of each servers. 427func DetectPlatforms(timeoutSec int) { 428 detectPlatforms(timeoutSec) 429 for i, s := range servers { 430 if s.getServerInfo().IsContainer() { 431 util.Log.Infof("(%d/%d) %s on %s is running on %s", 432 i+1, len(servers), 433 s.getServerInfo().Container.Name, 434 s.getServerInfo().ServerName, 435 s.getPlatform().Name, 436 ) 437 438 } else { 439 util.Log.Infof("(%d/%d) %s is running on %s", 440 i+1, len(servers), 441 s.getServerInfo().ServerName, 442 s.getPlatform().Name, 443 ) 444 } 445 } 446 return 447} 448 449func detectPlatforms(timeoutSec int) { 450 parallelExec(func(o osTypeInterface) error { 451 o.detectPlatform() 452 // Logging only if platform can not be specified 453 return nil 454 }, timeoutSec) 455 return 456} 457 458// DetectIPSs detects the IPS of each servers. 459func DetectIPSs(timeoutSec int) { 460 detectIPSs(timeoutSec) 461 for i, s := range servers { 462 if !s.getServerInfo().IsContainer() { 463 util.Log.Infof("(%d/%d) %s has %d IPS integration", 464 i+1, len(servers), 465 s.getServerInfo().ServerName, 466 len(s.getServerInfo().IPSIdentifiers), 467 ) 468 } 469 } 470} 471 472func detectIPSs(timeoutSec int) { 473 parallelExec(func(o osTypeInterface) error { 474 o.detectIPSs() 475 // Logging only if IPS can not be specified 476 return nil 477 }, timeoutSec) 478} 479 480// Scan scan 481func Scan(timeoutSec int) error { 482 if len(servers) == 0 { 483 return xerrors.New("No server defined. Check the configuration") 484 } 485 486 if err := setupChangelogCache(); err != nil { 487 return err 488 } 489 defer func() { 490 if cache.DB != nil { 491 cache.DB.Close() 492 } 493 }() 494 495 util.Log.Info("Scanning vulnerable OS packages...") 496 scannedAt := time.Now() 497 dir, err := EnsureResultDir(scannedAt) 498 if err != nil { 499 return err 500 } 501 502 results, err := GetScanResults(scannedAt, timeoutSec) 503 if err != nil { 504 return err 505 } 506 507 for i, r := range results { 508 if s, ok := config.Conf.Servers[r.ServerName]; ok { 509 results[i] = r.ClearFields(s.IgnoredJSONKeys) 510 } 511 } 512 513 return writeScanResults(dir, results) 514} 515 516// ViaHTTP scans servers by HTTP header and body 517func ViaHTTP(header http.Header, body string) (models.ScanResult, error) { 518 family := header.Get("X-Vuls-OS-Family") 519 if family == "" { 520 return models.ScanResult{}, errOSFamilyHeader 521 } 522 523 release := header.Get("X-Vuls-OS-Release") 524 if release == "" { 525 return models.ScanResult{}, errOSReleaseHeader 526 } 527 528 kernelRelease := header.Get("X-Vuls-Kernel-Release") 529 if kernelRelease == "" { 530 util.Log.Warn("If X-Vuls-Kernel-Release is not specified, there is a possibility of false detection") 531 } 532 533 kernelVersion := header.Get("X-Vuls-Kernel-Version") 534 if family == config.Debian && kernelVersion == "" { 535 return models.ScanResult{}, errKernelVersionHeader 536 } 537 538 serverName := header.Get("X-Vuls-Server-Name") 539 if config.Conf.ToLocalFile && serverName == "" { 540 return models.ScanResult{}, errServerNameHeader 541 } 542 543 distro := config.Distro{ 544 Family: family, 545 Release: release, 546 } 547 548 kernel := models.Kernel{ 549 Release: kernelRelease, 550 Version: kernelVersion, 551 } 552 base := base{ 553 Distro: distro, 554 osPackages: osPackages{ 555 Kernel: kernel, 556 }, 557 log: util.Log, 558 } 559 560 var osType osTypeInterface 561 switch family { 562 case config.Debian, config.Ubuntu: 563 osType = &debian{base: base} 564 case config.RedHat: 565 osType = &rhel{ 566 redhatBase: redhatBase{base: base}, 567 } 568 case config.CentOS: 569 osType = ¢os{ 570 redhatBase: redhatBase{base: base}, 571 } 572 case config.Amazon: 573 osType = &amazon{ 574 redhatBase: redhatBase{base: base}, 575 } 576 default: 577 return models.ScanResult{}, xerrors.Errorf("Server mode for %s is not implemented yet", family) 578 } 579 580 installedPackages, srcPackages, err := osType.parseInstalledPackages(body) 581 if err != nil { 582 return models.ScanResult{}, err 583 } 584 585 result := models.ScanResult{ 586 ServerName: serverName, 587 Family: family, 588 Release: release, 589 RunningKernel: models.Kernel{ 590 Release: kernelRelease, 591 Version: kernelVersion, 592 }, 593 Packages: installedPackages, 594 SrcPackages: srcPackages, 595 ScannedCves: models.VulnInfos{}, 596 } 597 598 return result, nil 599} 600 601func setupChangelogCache() error { 602 needToSetupCache := false 603 for _, s := range servers { 604 switch s.getDistro().Family { 605 case config.Raspbian: 606 needToSetupCache = true 607 break 608 case config.Ubuntu, config.Debian: 609 //TODO changelog cache for RedHat, Oracle, Amazon, CentOS is not implemented yet. 610 if s.getServerInfo().Mode.IsDeep() { 611 needToSetupCache = true 612 } 613 break 614 } 615 } 616 if needToSetupCache { 617 if err := cache.SetupBolt(config.Conf.CacheDBPath, util.Log); err != nil { 618 return err 619 } 620 } 621 return nil 622} 623 624// GetScanResults returns ScanResults from 625func GetScanResults(scannedAt time.Time, timeoutSec int) (results models.ScanResults, err error) { 626 parallelExec(func(o osTypeInterface) (err error) { 627 if !(config.Conf.LibsOnly || config.Conf.WordPressOnly) { 628 if err = o.preCure(); err != nil { 629 return err 630 } 631 if err = o.scanPackages(); err != nil { 632 return err 633 } 634 if err = o.postScan(); err != nil { 635 return err 636 } 637 } 638 if err = o.scanWordPress(); err != nil { 639 return xerrors.Errorf("Failed to scan WordPress: %w", err) 640 } 641 if err = o.scanLibraries(); err != nil { 642 return xerrors.Errorf("Failed to scan Library: %w", err) 643 } 644 if err = o.scanPorts(); err != nil { 645 return xerrors.Errorf("Failed to scan Ports: %w", err) 646 } 647 return nil 648 }, timeoutSec) 649 650 hostname, _ := os.Hostname() 651 ipv4s, ipv6s, err := util.IP() 652 if err != nil { 653 util.Log.Errorf("Failed to fetch scannedIPs. err: %+v", err) 654 } 655 656 for _, s := range append(servers, errServers...) { 657 r := s.convertToModel() 658 r.ScannedAt = scannedAt 659 r.ScannedVersion = config.Version 660 r.ScannedRevision = config.Revision 661 r.ScannedBy = hostname 662 r.ScannedIPv4Addrs = ipv4s 663 r.ScannedIPv6Addrs = ipv6s 664 r.Config.Scan = config.Conf 665 results = append(results, r) 666 667 if 0 < len(r.Warnings) { 668 util.Log.Warnf("Some warnings occurred during scanning on %s. Please fix the warnings to get a useful information. Execute configtest subcommand before scanning to know the cause of the warnings. warnings: %v", 669 r.ServerName, r.Warnings) 670 } 671 } 672 return results, nil 673} 674 675func writeScanResults(jsonDir string, results models.ScanResults) error { 676 config.Conf.FormatJSON = true 677 ws := []report.ResultWriter{ 678 report.LocalFileWriter{CurrentDir: jsonDir}, 679 } 680 for _, w := range ws { 681 if err := w.Write(results...); err != nil { 682 return xerrors.Errorf("Failed to write summary report: %s", err) 683 } 684 } 685 686 report.StdoutWriter{}.WriteScanSummary(results...) 687 688 errServerNames := []string{} 689 for _, r := range results { 690 if 0 < len(r.Errors) { 691 errServerNames = append(errServerNames, r.ServerName) 692 } 693 } 694 if 0 < len(errServerNames) { 695 return fmt.Errorf("An error occurred on %s", errServerNames) 696 } 697 return nil 698} 699 700// EnsureResultDir ensures the directory for scan results 701func EnsureResultDir(scannedAt time.Time) (currentDir string, err error) { 702 jsonDirName := scannedAt.Format(time.RFC3339) 703 704 resultsDir := config.Conf.ResultsDir 705 if len(resultsDir) == 0 { 706 wd, _ := os.Getwd() 707 resultsDir = filepath.Join(wd, "results") 708 } 709 jsonDir := filepath.Join(resultsDir, jsonDirName) 710 if err := os.MkdirAll(jsonDir, 0700); err != nil { 711 return "", xerrors.Errorf("Failed to create dir: %w", err) 712 } 713 714 symlinkPath := filepath.Join(resultsDir, "current") 715 if _, err := os.Lstat(symlinkPath); err == nil { 716 if err := os.Remove(symlinkPath); err != nil { 717 return "", xerrors.Errorf( 718 "Failed to remove symlink. path: %s, err: %w", symlinkPath, err) 719 } 720 } 721 722 if err := os.Symlink(jsonDir, symlinkPath); err != nil { 723 return "", xerrors.Errorf( 724 "Failed to create symlink: path: %s, err: %w", symlinkPath, err) 725 } 726 return jsonDir, nil 727} 728