1 /*
2 * Copyright (c) 2014-2020 by Farsight Security, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 /* asprintf() does not appear on linux without this */
18 #define _GNU_SOURCE
19
20 #define _BSD_SOURCE
21 #define _DEFAULT_SOURCE
22
23 #include <sys/wait.h>
24
25 #include <assert.h>
26 #include <errno.h>
27 #include <signal.h>
28 #include <string.h>
29 #include <unistd.h>
30
31 #include "defs.h"
32 #include "netio.h"
33 #include "pdns.h"
34 #include "globals.h"
35
36 static void io_drain(void);
37 static void fetch_reap(fetch_t);
38 static void fetch_done(fetch_t);
39 static void fetch_unlink(fetch_t);
40 static void query_done(query_t);
41
42 static writer_t writers = NULL;
43 static CURLM *multi = NULL;
44 static bool curl_cleanup_needed = false;
45
46 const char saf_begin[] = "begin";
47 const char saf_ongoing[] = "ongoing";
48 const char saf_succeeded[] = "succeeded";
49 const char saf_limited[] = "limited";
50 const char saf_failed[] = "failed";
51
52 const char *saf_valid_conds[] = {
53 saf_begin, saf_ongoing, saf_succeeded, saf_limited, saf_failed
54 };
55
56 /* make_curl -- perform global initializations of libcurl.
57 */
58 void
make_curl(void)59 make_curl(void) {
60 curl_global_init(CURL_GLOBAL_DEFAULT);
61 curl_cleanup_needed = true;
62 multi = curl_multi_init();
63 if (multi == NULL) {
64 fprintf(stderr, "%s: curl_multi_init() failed\n",
65 program_name);
66 my_exit(1);
67 }
68 }
69
70 /* unmake_curl -- clean up and discard libcurl's global state.
71 */
72 void
unmake_curl(void)73 unmake_curl(void) {
74 if (multi != NULL) {
75 curl_multi_cleanup(multi);
76 multi = NULL;
77 }
78 if (curl_cleanup_needed) {
79 curl_global_cleanup();
80 curl_cleanup_needed = false;
81 }
82 }
83
84 /* fetch -- given a url, tell libcurl to go fetch it.
85 */
86 void
create_fetch(query_t query,char * url)87 create_fetch(query_t query, char *url) {
88 fetch_t fetch = NULL;
89 CURLMcode res;
90
91 DEBUG(2, true, "fetch(%s)\n", url);
92 CREATE(fetch, sizeof *fetch);
93 fetch->query = query;
94 query = NULL;
95 fetch->easy = curl_easy_init();
96 if (fetch->easy == NULL) {
97 /* an error will have been output by libcurl in this case. */
98 DESTROY(fetch);
99 DESTROY(url);
100 my_exit(1);
101 }
102 fetch->url = url;
103 url = NULL;
104 curl_easy_setopt(fetch->easy, CURLOPT_URL, fetch->url);
105 if (donotverify) {
106 curl_easy_setopt(fetch->easy, CURLOPT_SSL_VERIFYPEER, 0L);
107 curl_easy_setopt(fetch->easy, CURLOPT_SSL_VERIFYHOST, 0L);
108 }
109
110 /* if user specified a prefence for IPv4 or IPv6, use it. */
111 if (curl_ipresolve != CURL_IPRESOLVE_WHATEVER)
112 curl_easy_setopt(fetch->easy,
113 CURLOPT_IPRESOLVE, curl_ipresolve);
114
115 if (psys->auth != NULL)
116 psys->auth(fetch);
117
118 fetch->hdrs = curl_slist_append(fetch->hdrs, jsonl_header);
119 curl_easy_setopt(fetch->easy, CURLOPT_HTTPHEADER, fetch->hdrs);
120 curl_easy_setopt(fetch->easy, CURLOPT_WRITEFUNCTION, writer_func);
121 curl_easy_setopt(fetch->easy, CURLOPT_WRITEDATA, fetch);
122 curl_easy_setopt(fetch->easy, CURLOPT_PRIVATE, fetch);
123 #ifdef CURL_AT_LEAST_VERSION
124 /* If CURL_AT_LEAST_VERSION is not defined then the curl is probably too old */
125 #if CURL_AT_LEAST_VERSION(7,42,0)
126 /* do not allow curl to swallow /./ and /../ in our URLs */
127 curl_easy_setopt(fetch->easy, CURLOPT_PATH_AS_IS, 1L);
128 #endif
129 #endif /* CURL_AT_LEAST_VERSION */
130 if (debug_level >= 3)
131 curl_easy_setopt(fetch->easy, CURLOPT_VERBOSE, 1L);
132
133 fetch->query->fetch = fetch;
134
135 res = curl_multi_add_handle(multi, fetch->easy);
136 if (res != CURLM_OK) {
137 fprintf(stderr, "%s: curl_multi_add_handle() failed: %s\n",
138 program_name, curl_multi_strerror(res));
139 my_exit(1);
140 }
141 }
142
143 /* fetch_reap -- reap one fetch.
144 */
145 static void
fetch_reap(fetch_t fetch)146 fetch_reap(fetch_t fetch) {
147 if (fetch->easy != NULL) {
148 curl_multi_remove_handle(multi, fetch->easy);
149 curl_easy_cleanup(fetch->easy);
150 fetch->easy = NULL;
151 }
152 if (fetch->hdrs != NULL) {
153 curl_slist_free_all(fetch->hdrs);
154 fetch->hdrs = NULL;
155 }
156 DESTROY(fetch->url);
157 DESTROY(fetch->buf);
158 DESTROY(fetch);
159 }
160
161 /* fetch_done -- deal with consequences of end-of-fetch.
162 */
163 static void
fetch_done(fetch_t fetch)164 fetch_done(fetch_t fetch) {
165 query_t query = fetch->query;
166
167 query_done(query);
168 }
169
170 /* fetch_unlink -- disconnect a fetch from its writer.
171 */
172 static void
fetch_unlink(fetch_t fetch)173 fetch_unlink(fetch_t fetch) {
174 assert(fetch == fetch->query->fetch);
175 fetch->query->fetch = NULL;
176 fetch->query = NULL;
177 }
178
179 /* writer_init -- instantiate a writer
180 */
181 writer_t
writer_init(long output_limit)182 writer_init(long output_limit) {
183 writer_t writer = NULL;
184
185 CREATE(writer, sizeof(struct writer));
186 writer->output_limit = output_limit;
187
188 return (writer);
189 }
190
191 /* query_status -- install a status code and description in a query.
192 */
193 void
query_status(query_t query,const char * status,const char * message)194 query_status(query_t query, const char *status, const char *message) {
195 assert((query->status == NULL) == (query->message == NULL));
196 assert(query->status == NULL);
197 query->status = strdup(status);
198 query->message = strdup(message);
199 }
200
201 /* writer_func -- process a block of json text.
202 *
203 * This function's signature must conform to write_callback() in
204 * CURLOPT_WRITEFUNCTION.
205 * Returns the number of bytes actually taken care of or returns
206 * CURL_WRITEFUNC_PAUSE to pause this query's connection until
207 * curl_easy_pause(..., CURLPAUSE_CONT) is called.
208 */
209 size_t
writer_func(char * ptr,size_t size,size_t nmemb,void * blob)210 writer_func(char *ptr, size_t size, size_t nmemb, void *blob) {
211 fetch_t fetch = (fetch_t) blob;
212 query_t query = fetch->query;
213 writer_t writer = query->writer;
214 size_t bytes = size * nmemb;
215 char *nl;
216
217 DEBUG(3, true, "writer_func(%d, %d): %d\n",
218 (int)size, (int)nmemb, (int)bytes);
219
220 fetch->buf = realloc(fetch->buf, fetch->len + bytes);
221 memcpy(fetch->buf + fetch->len, ptr, bytes);
222 fetch->len += bytes;
223
224 /* when the fetch is a live web result, emit
225 * !2xx errors and info payloads as reports.
226 */
227 if (fetch->easy != NULL) {
228 if (fetch->rcode == 0)
229 curl_easy_getinfo(fetch->easy,
230 CURLINFO_RESPONSE_CODE,
231 &fetch->rcode);
232 if (fetch->rcode != HTTP_OK) {
233 char *message = strndup(fetch->buf, fetch->len);
234
235 /* only report the first line of data. */
236 char *eol = strpbrk(message, "\r\n");
237 if (eol != NULL)
238 *eol = '\0';
239
240 /* if the message (that's left) is just <html>,
241 * change it to an HTTP Status code */
242 if (strcasecmp(message, "<html>") == 0) {
243 DESTROY(message);
244 if (asprintf(&message, "HTTP Status %ld",
245 fetch->rcode) < 0)
246 my_panic(true, "asprintf");
247 }
248
249 /* only report the first response status (vs. -m). */
250 if (query->status == NULL) {
251 query_status(query,
252 psys->status(fetch),
253 message);
254 if (!quiet) {
255 char *url;
256
257 curl_easy_getinfo(fetch->easy,
258 CURLINFO_EFFECTIVE_URL,
259 &url);
260 fprintf(stderr,
261 "%s: warning: "
262 "libcurl %ld [%s]\n",
263 program_name, fetch->rcode,
264 url);
265 }
266 }
267 if (!quiet)
268 fprintf(stderr, "%s: warning: libcurl: [%s]\n",
269 program_name, message);
270 DESTROY(message);
271 fetch->buf[0] = '\0';
272 fetch->len = 0;
273 return (bytes);
274 }
275 }
276
277 /* deblock. */
278 while ((nl = memchr(fetch->buf, '\n', fetch->len)) != NULL) {
279 size_t pre_len = (size_t)(nl - fetch->buf),
280 post_len = (fetch->len - pre_len) - 1;
281
282 if (writer->output_limit > 0 &&
283 writer->count >= writer->output_limit)
284 {
285 DEBUG(9, true, "hit output limit %ld\n",
286 writer->output_limit);
287 /* cause CURLE_WRITE_ERROR for this transfer. */
288 bytes = 0;
289 query->saf_cond = sc_we_limited;
290 /* inform io_engine() that the abort is intentional. */
291 fetch->stopped = true;
292 } else {
293 query->writer->count +=
294 data_blob(query, fetch->buf, pre_len);
295
296 switch (query->saf_cond) {
297 case sc_init:
298 case sc_begin:
299 case sc_ongoing:
300 case sc_missing:
301 break;
302 case sc_succeeded:
303 case sc_limited:
304 case sc_failed:
305 case sc_we_limited:
306 /* inform io_engine() intentional
307 * abort. */
308 fetch->stopped = true;
309 break;
310 }
311 }
312 memmove(fetch->buf, nl + 1, post_len);
313 fetch->len = post_len;
314 }
315
316 return (bytes);
317 }
318
319 /* query_done -- do something with leftover buffer data when a query ends.
320 */
321 static void
query_done(query_t query)322 query_done(query_t query) {
323 DEBUG(2, true, "query_done(%s)\n", query->command);
324
325 if (!quiet) {
326 const char *msg = or_else(query->saf_msg, "");
327
328 if (query->saf_cond == sc_limited)
329 fprintf(stderr, "Query limited: %s\n", msg);
330 else if (query->saf_cond == sc_failed)
331 fprintf(stderr, "Query failed: %s\n", msg);
332 else if (query->saf_cond == sc_missing)
333 fprintf(stderr, "Query response_missing: %s\n", msg);
334 else if (query->status != NULL)
335 fprintf(stderr, "Query status: %s (%s)\n",
336 query->status, query->message);
337 }
338 }
339
340 /* writer_fini -- stop a writer's fetch
341 */
342 void
writer_fini(writer_t writer)343 writer_fini(writer_t writer) {
344 /* finish and close any fetches still cooking. */
345 if (writer->query != NULL) {
346 query_t query = writer->query;
347
348 /* release any buffered info. */
349 if (query->fetch != NULL) {
350 DESTROY(query->fetch->buf);
351 if (query->fetch->len != 0) {
352 fprintf(stderr,
353 "%s: warning: stranding %d octets!\n",
354 program_name, (int)query->fetch->len);
355 query->fetch->len = 0;
356 }
357
358 /* tear down any curl infrastructure on the fetch. */
359 fetch_reap(query->fetch);
360
361 query->fetch = NULL;
362 }
363 assert((query->status != NULL) == (query->message != NULL));
364 DESTROY(query->status);
365 DESTROY(query->message);
366 DESTROY(query->command);
367 DESTROY(query);
368 }
369
370 DESTROY(writer);
371 }
372
373 void
unmake_writers(void)374 unmake_writers(void) {
375 while (writers != NULL)
376 writer_fini(writers);
377 }
378
379 /* io_engine -- let libcurl run until there are few enough outstanding jobs.
380 */
381 void
io_engine(int jobs)382 io_engine(int jobs) {
383 int still, repeats, numfds;
384
385 DEBUG(2, true, "io_engine(%d)\n", jobs);
386
387 /* let libcurl run while there are too many jobs remaining. */
388 still = 0;
389 repeats = 0;
390 while (curl_multi_perform(multi, &still) == CURLM_OK && still > jobs) {
391 DEBUG(3, true, "...waiting (still %d)\n", still);
392 numfds = 0;
393 if (curl_multi_wait(multi, NULL, 0, 0, &numfds) != CURLM_OK)
394 break;
395 if (numfds == 0) {
396 /* curl_multi_wait() can return 0 fds for no reason. */
397 if (++repeats > 1) {
398 struct timespec req, rem;
399
400 req = (struct timespec){
401 .tv_sec = 0,
402 .tv_nsec = 100*1000*1000 // 100ms
403 };
404 while (nanosleep(&req, &rem) == EINTR) {
405 /* as required by nanosleep(3). */
406 req = rem;
407 }
408 }
409 } else {
410 repeats = 0;
411 }
412 io_drain();
413 }
414 io_drain();
415 }
416
417 /* io_drain -- drain the response code reports.
418 */
419 static void
io_drain(void)420 io_drain(void) {
421 struct CURLMsg *cm;
422 int still = 0;
423
424 while ((cm = curl_multi_info_read(multi, &still)) != NULL) {
425 fetch_t fetch;
426 query_t query;
427 char *private;
428
429 curl_easy_getinfo(cm->easy_handle,
430 CURLINFO_PRIVATE,
431 &private);
432 fetch = (fetch_t) private;
433 query = fetch->query;
434
435 if (cm->msg == CURLMSG_DONE) {
436 if (fetch->rcode == 0)
437 curl_easy_getinfo(fetch->easy,
438 CURLINFO_RESPONSE_CODE,
439 &fetch->rcode);
440
441 DEBUG(2, true, "io_drain(%s) DONE rcode=%d\n",
442 query->command, fetch->rcode);
443 DEBUG(2, true, "... saf_cond %d saf_msg %s\n",
444 query->saf_cond,
445 or_else(query->saf_msg, ""));
446
447 if (cm->data.result == CURLE_COULDNT_RESOLVE_HOST) {
448 fprintf(stderr,
449 "%s: warning: libcurl failed since "
450 "could not resolve host\n",
451 program_name);
452 exit_code = 1;
453 } else if (cm->data.result == CURLE_COULDNT_CONNECT) {
454 fprintf(stderr,
455 "%s: warning: libcurl failed since "
456 "could not connect\n",
457 program_name);
458 exit_code = 1;
459 } else if (cm->data.result != CURLE_OK &&
460 !fetch->stopped)
461 {
462 fprintf(stderr,
463 "%s: warning: libcurl failed with "
464 "curl error %d (%s)\n",
465 program_name, cm->data.result,
466 curl_easy_strerror(cm->data.result));
467 exit_code = 1;
468 }
469
470 /* record emptiness as status if nothing else. */
471 if (query->writer != NULL &&
472 query->writer->count == 0 &&
473 query->status == NULL)
474 {
475 query_status(query,
476 status_noerror,
477 "no results found for query.");
478 }
479
480 fetch_done(fetch);
481 fetch_unlink(fetch);
482 fetch_reap(fetch);
483 }
484 DEBUG(3, true, "...info read (still %d)\n", still);
485 }
486 }
487
488 /* escape -- HTML-encode a string, in place.
489 */
490 void
escape(CURL * easy,char ** str)491 escape(CURL *easy, char **str) {
492 char *escaped;
493
494 if (*str == NULL)
495 return;
496 escaped = curl_easy_escape(easy, *str, (int)strlen(*str));
497 if (escaped == NULL) {
498 fprintf(stderr, "%s: curl_escape(%s) failed\n",
499 program_name, *str);
500 my_exit(1);
501 }
502 DESTROY(*str);
503 *str = strdup(escaped);
504 curl_free(escaped);
505 escaped = NULL;
506 }
507