1 /***************************************************************************/ 2 /* This code is part of WWW grabber called pavuk */ 3 /* Copyright (c) 1997 - 2001 Stefan Ondrejicka */ 4 /* Distributed under GPL 2 or later */ 5 /***************************************************************************/ 6 7 #ifndef _robots_h_ 8 #define _robots_h_ 9 10 #include "tools.h" 11 #include "mt.h" 12 #include "url.h" 13 14 typedef struct 15 { 16 char *site; /*** host name of server for "robots.txt" checking ***/ 17 int port; /*** server port ***/ 18 char **dpat; /*** list of disallowed prefixes ***/ 19 char **apat; /*** list of allowed prefixes */ 20 #ifdef HAVE_MT 21 pthread_mutex_t lock; /*** lock to prevent downloading concurrently with multiple threads ***/ 22 #endif 23 } robotlim; 24 25 extern bool_t robots_check(url *); 26 extern void robots_do_cleanup(void); 27 28 #endif 29