Searched defs:RobotsData (Results 1 – 3 of 3) sorted by relevance
/dports/security/keybase/client-v5.7.1/go/vendor/github.com/temoto/robotstxt/ |
H A D | robotstxt.go | 20 type RobotsData struct { struct 22 groups map[string]*Group 23 allowAll bool 24 disallowAll bool 25 Host string 26 Sitemaps []string 140 func (r *RobotsData) TestAgent(path, agent string) bool { 162 func (r *RobotsData) FindGroup(agent string) (ret *Group) {
|
/dports/www/colly/colly-2.1.0/vendor/github.com/temoto/robotstxt/ |
H A D | robotstxt.go | 20 type RobotsData struct { struct 22 groups map[string]*Group 23 allowAll bool 24 disallowAll bool 25 Host string 26 Sitemaps []string 136 func (r *RobotsData) TestAgent(path, agent string) bool { 158 func (r *RobotsData) FindGroup(agent string) (ret *Group) {
|
H A D | robotstxt_test.go | 238 func expectAll(t *testing.T, r *RobotsData, allow bool) { 246 func expectAllAgents(t *testing.T, r *RobotsData, allow bool, path string) { 253 func expectAccess(t *testing.T, r *RobotsData, allow bool, path, agent string) bool {
|