1 // Larbin
2 // Sebastien Ailleret
3 // 03-02-00 -> 10-12-01
4
5 #include <iostream>
6 #include <cstring>
7 #include <unistd.h>
8
9 #include "options.h"
10
11 #include "types.h"
12 #include "global.h"
13 #include "fetch/file.h"
14 #include "utils/text.h"
15 #include "utils/debug.h"
16 #include "interf/useroutput.h"
17 #include "utils/mypthread.h"
18
19
20 /** The fetch failed
21 * @param u the URL of the doc
22 * @param reason reason of the fail
23 */
fetchFail(url * u,FetchError err,bool interesting=false)24 void fetchFail (url *u, FetchError err, bool interesting=false) {
25 #ifdef SPECIFICSEARCH
26 if (interesting
27 || (privilegedExts[0] != NULL && matchPrivExt(u->getFile()))) {
28 failure(u, err);
29 }
30 #else // not a SPECIFICSEARCH
31 failure(u, err);
32 #endif
33 }
34
35 /** It's over with this file
36 * report the situation ! (and make some stats)
37 */
endOfLoad(html * parser,FetchError err)38 void endOfLoad (html *parser, FetchError err) {
39 answers(err);
40 switch (err) {
41 case success:
42 #ifdef SPECIFICSEARCH
43 if (parser->isInteresting) {
44 interestingPage();
45 loaded(parser);
46 }
47 #else // not a SPECIFICSEARCH
48 loaded(parser);
49 #endif // SPECIFICSEARCH
50 break;
51 default:
52 fetchFail(parser->getUrl(), err, parser->isInteresting);
53 break;
54 }
55 }
56
57 #ifdef THREAD_OUTPUT
58 /** In this thread, end user manage the result of the crawl
59 */
startOutput(void * none)60 static void *startOutput (void *none) {
61 initUserOutput();
62 for (;;) {
63 Connexion *conn = global::userConns->get();
64 endOfLoad((html *)conn->parser, conn->err);
65 conn->recycle();
66 global::freeConns->put(conn);
67 }
68 return NULL;
69 }
70
initOutput()71 void initOutput () {
72 startThread(startOutput, NULL);
73 }
74
75 #else // THREAD_OUTPUT not defined
76
initOutput()77 void initOutput () {
78 initUserOutput();
79 }
80
81 #endif // THREAD_OUTPUT
82