1package main
2
3import (
4	"flag"
5	"log"
6	"net/http"
7	"strings"
8
9	"github.com/temoto/robotstxt"
10)
11
12var checkPaths = []string{
13	"/",
14}
15
16func main() {
17	robotsUrl := flag.String("robots-url", "", "")
18	bot := flag.String("bot", "GoogleBot", "")
19	flag.Parse()
20	if *robotsUrl == "" {
21		log.Fatalln("Robots URL is empty, run with -h to see usage.")
22	}
23	if !strings.HasPrefix(*robotsUrl, "http") {
24		*robotsUrl = "http://" + *robotsUrl
25	}
26
27	response, err := http.Get(*robotsUrl)
28	if err != nil {
29		log.Fatalln("HTTP error:", err)
30	}
31
32	robots, err := robotstxt.FromResponse(response)
33	if err != nil {
34		log.Fatalln("Robots.txt error:", err)
35	}
36
37	log.Println("Running checks as", *bot)
38	group := robots.FindGroup(*bot)
39	for _, path := range checkPaths {
40		log.Println(path, ":", group.Test(path))
41	}
42}
43