1 /***************************************************************************/
2 /*    This code is part of WWW grabber called pavuk                        */
3 /*    Copyright (c) 1997 - 2001 Stefan Ondrejicka                          */
4 /*    Distributed under GPL 2 or later                                     */
5 /***************************************************************************/
6 
7 #ifndef _robots_h_
8 #define _robots_h_
9 
10 #include "tools.h"
11 #include "mt.h"
12 #include "url.h"
13 
14 typedef struct
15 {
16   char *site;                   /*** host name of server for "robots.txt" checking ***/
17   int port;                     /*** server port ***/
18   char **dpat;                  /*** list of disallowed prefixes ***/
19   char **apat;                  /*** list of allowed prefixes */
20 #ifdef HAVE_MT
21   pthread_mutex_t lock;         /*** lock to prevent downloading concurrently with multiple threads ***/
22 #endif
23 } robotlim;
24 
25 extern bool_t robots_check(url *);
26 extern void robots_do_cleanup(void);
27 
28 #endif
29