1package main 2 3import ( 4 "flag" 5 "log" 6 "net/http" 7 "strings" 8 9 "github.com/temoto/robotstxt" 10) 11 12var checkPaths = []string{ 13 "/", 14} 15 16func main() { 17 robotsUrl := flag.String("robots-url", "", "") 18 bot := flag.String("bot", "GoogleBot", "") 19 flag.Parse() 20 if *robotsUrl == "" { 21 log.Fatalln("Robots URL is empty, run with -h to see usage.") 22 } 23 if !strings.HasPrefix(*robotsUrl, "http") { 24 *robotsUrl = "http://" + *robotsUrl 25 } 26 27 response, err := http.Get(*robotsUrl) 28 if err != nil { 29 log.Fatalln("HTTP error:", err) 30 } 31 32 robots, err := robotstxt.FromResponse(response) 33 if err != nil { 34 log.Fatalln("Robots.txt error:", err) 35 } 36 37 log.Println("Running checks as", *bot) 38 group := robots.FindGroup(*bot) 39 for _, path := range checkPaths { 40 log.Println(path, ":", group.Test(path)) 41 } 42} 43