1 /**
2 * @file net.c HTTP network access using libsoup
3 *
4 * Copyright (C) 2007-2015 Lars Windolf <lars.windolf@gmx.de>
5 * Copyright (C) 2009 Emilio Pozuelo Monfort <pochu27@gmail.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 */
21
22 #include "net.h"
23
24 #include <glib.h>
25 #include <libsoup/soup.h>
26 #include <math.h>
27 #include <string.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <time.h>
31
32 #include "common.h"
33 #include "conf.h"
34 #include "debug.h"
35
36 #define HOMEPAGE "https://lzone.de/liferea/"
37
38 static SoupSession *session = NULL; /* Session configured for preferences */
39 static SoupSession *session2 = NULL; /* Session for "Don't use proxy feature" */
40
41 static ProxyDetectMode proxymode = PROXY_DETECT_MODE_AUTO;
42 static gchar *proxyname = NULL;
43 static gchar *proxyusername = NULL;
44 static gchar *proxypassword = NULL;
45 static int proxyport = 0;
46
47
48 static void
network_process_redirect_callback(SoupMessage * msg,gpointer user_data)49 network_process_redirect_callback (SoupMessage *msg, gpointer user_data)
50 {
51 updateJobPtr job = (updateJobPtr)user_data;
52 const gchar *location = NULL;
53 SoupURI *newuri;
54
55 if (301 == msg->status_code || 308 == msg->status_code)
56 {
57 location = soup_message_headers_get_one (msg->response_headers, "Location");
58 newuri = soup_uri_new (location);
59
60 if (SOUP_URI_IS_VALID (newuri) && ! soup_uri_equal (newuri, soup_message_get_uri (msg))) {
61 debug2 (DEBUG_NET, "\"%s\" permanently redirects to new location \"%s\"", soup_uri_to_string (soup_message_get_uri (msg), FALSE),
62 soup_uri_to_string (newuri, FALSE));
63 job->result->returncode = msg->status_code;
64 job->result->source = soup_uri_to_string (newuri, FALSE);
65 }
66 }
67 }
68
69 static void
network_process_callback(SoupSession * session,SoupMessage * msg,gpointer user_data)70 network_process_callback (SoupSession *session, SoupMessage *msg, gpointer user_data)
71 {
72 updateJobPtr job = (updateJobPtr)user_data;
73 SoupDate *last_modified;
74 const gchar *tmp = NULL;
75 GHashTable *params;
76 gboolean revalidated = FALSE;
77 gint maxage;
78 gint age;
79
80 job->result->source = soup_uri_to_string (soup_message_get_uri(msg), FALSE);
81 if (SOUP_STATUS_IS_TRANSPORT_ERROR (msg->status_code)) {
82 job->result->returncode = msg->status_code;
83 job->result->httpstatus = 0;
84 } else {
85 job->result->httpstatus = msg->status_code;
86 }
87
88 /* keep some request headers for revalidated responses */
89 revalidated = (304 == job->result->httpstatus);
90
91 debug1 (DEBUG_NET, "download status code: %d", msg->status_code);
92 debug1 (DEBUG_NET, "source after download: >>>%s<<<", job->result->source);
93
94 job->result->data = g_memdup (msg->response_body->data, msg->response_body->length+1);
95 job->result->size = (size_t)msg->response_body->length;
96 debug1 (DEBUG_NET, "%d bytes downloaded", job->result->size);
97
98 job->result->contentType = g_strdup (soup_message_headers_get_content_type (msg->response_headers, NULL));
99
100 /* Update last-modified date */
101 if (revalidated) {
102 job->result->updateState->lastModified = update_state_get_lastmodified (job->request->updateState);
103 } else {
104 tmp = soup_message_headers_get_one (msg->response_headers, "Last-Modified");
105 if (tmp) {
106 /* The string may be badly formatted, which will make
107 * soup_date_new_from_string() return NULL */
108 last_modified = soup_date_new_from_string (tmp);
109 if (last_modified) {
110 job->result->updateState->lastModified = soup_date_to_time_t (last_modified);
111 soup_date_free (last_modified);
112 }
113 }
114 }
115
116 /* Update ETag value */
117 if (revalidated) {
118 job->result->updateState->etag = g_strdup (update_state_get_etag (job->request->updateState));
119 } else {
120 tmp = soup_message_headers_get_one (msg->response_headers, "ETag");
121 if (tmp) {
122 job->result->updateState->etag = g_strdup (tmp);
123 }
124 }
125
126 /* Update cache max-age */
127 tmp = soup_message_headers_get_list (msg->response_headers, "Cache-Control");
128 if (tmp) {
129 params = soup_header_parse_param_list (tmp);
130 if (params) {
131 tmp = g_hash_table_lookup (params, "max-age");
132 if (tmp) {
133 maxage = atoi (tmp);
134 if (0 < maxage) {
135 /* subtract Age from max-age */
136 tmp = soup_message_headers_get_one (msg->response_headers, "Age");
137 if (tmp) {
138 age = atoi (tmp);
139 if (0 < age) {
140 maxage = maxage - age;
141 }
142 }
143 if (0 < maxage) {
144 job->result->updateState->maxAgeMinutes = ceil ( (float) (maxage / 60));
145 }
146 }
147 }
148 }
149 soup_header_free_param_list (params);
150 }
151
152 update_process_finished_job (job);
153 }
154
155 static SoupURI *
network_get_proxy_uri(void)156 network_get_proxy_uri (void)
157 {
158 SoupURI *uri = NULL;
159
160 if (!proxyname)
161 return uri;
162
163 uri = soup_uri_new (NULL);
164 soup_uri_set_scheme (uri, SOUP_URI_SCHEME_HTTP);
165 soup_uri_set_host (uri, proxyname);
166 soup_uri_set_port (uri, proxyport);
167 soup_uri_set_user (uri, proxyusername);
168 soup_uri_set_password (uri, proxypassword);
169
170 return uri;
171 }
172
173 /* Downloads a feed specified in the request structure, returns
174 the downloaded data or NULL in the request structure.
175 If the webserver reports a permanent redirection, the
176 feed url will be modified and the old URL 'll be freed. The
177 request structure will also contain the HTTP status and the
178 last modified string.
179 */
180 void
network_process_request(const updateJobPtr job)181 network_process_request (const updateJobPtr job)
182 {
183 SoupMessage *msg;
184 SoupDate *date;
185 gboolean do_not_track = FALSE;
186
187 g_assert (NULL != job->request);
188 debug1 (DEBUG_NET, "downloading %s", job->request->source);
189 if (job->request->postdata && (debug_level & DEBUG_VERBOSE) && (debug_level & DEBUG_NET))
190 debug1 (DEBUG_NET, " postdata=>>>%s<<<", job->request->postdata);
191
192 /* Prepare the SoupMessage */
193 msg = soup_message_new (job->request->postdata ? SOUP_METHOD_POST : SOUP_METHOD_GET,
194 job->request->source);
195
196 if (!msg) {
197 g_warning ("The request for %s could not be parsed!", job->request->source);
198 return;
199 }
200
201 /* Set the postdata for the request */
202 if (job->request->postdata) {
203 soup_message_set_request (msg,
204 "application/x-www-form-urlencoded",
205 SOUP_MEMORY_STATIC, /* libsoup won't free the postdata */
206 job->request->postdata,
207 strlen (job->request->postdata));
208 }
209
210 /* Set the If-Modified-Since: header */
211 if (job->request->updateState && update_state_get_lastmodified (job->request->updateState)) {
212 gchar *datestr;
213
214 date = soup_date_new_from_time_t (update_state_get_lastmodified (job->request->updateState));
215 datestr = soup_date_to_string (date, SOUP_DATE_HTTP);
216 soup_message_headers_append (msg->request_headers,
217 "If-Modified-Since",
218 datestr);
219 g_free (datestr);
220 soup_date_free (date);
221 }
222
223 /* Set the If-None-Match header */
224 if (job->request->updateState && update_state_get_etag (job->request->updateState)) {
225 soup_message_headers_append(msg->request_headers,
226 "If-None-Match",
227 update_state_get_etag (job->request->updateState));
228 }
229
230 /* Set the I-AM header */
231 if (job->request->updateState &&
232 (update_state_get_lastmodified (job->request->updateState) ||
233 update_state_get_etag (job->request->updateState))) {
234 soup_message_headers_append(msg->request_headers,
235 "A-IM",
236 "feed");
237 }
238
239 /* Support HTTP content negotiation */
240 soup_message_headers_append(msg->request_headers, "Accept", "application/atom+xml,application/xml;q=0.9,text/xml;q=0.8,*/*;q=0.7");
241
242 /* Set the authentication */
243 if (!job->request->authValue &&
244 job->request->options &&
245 job->request->options->username &&
246 job->request->options->password) {
247 SoupURI *uri = soup_message_get_uri (msg);
248
249 soup_uri_set_user (uri, job->request->options->username);
250 soup_uri_set_password (uri, job->request->options->password);
251 }
252
253 if (job->request->authValue) {
254 soup_message_headers_append (msg->request_headers, "Authorization",
255 job->request->authValue);
256 }
257
258 /* Add requested cookies */
259 if (job->request->updateState && job->request->updateState->cookies) {
260 soup_message_headers_append (msg->request_headers, "Cookie",
261 job->request->updateState->cookies);
262 soup_message_disable_feature (msg, SOUP_TYPE_COOKIE_JAR);
263 }
264
265 /* TODO: Right now we send the msg, and if it requires authentication and
266 * we didn't provide one, the petition fails and when the job is processed
267 * it sees it needs authentication and displays a dialog, and if credentials
268 * are entered, it queues a new job with auth credentials. Instead of that,
269 * we should probably handle authentication directly here, connecting the
270 * msg to a callback in case of 401 (see soup_message_add_status_code_handler())
271 * displaying the dialog ourselves, and requeing the msg if we get credentials */
272
273 /* Add Do Not Track header according to settings */
274 conf_get_bool_value (DO_NOT_TRACK, &do_not_track);
275 if (do_not_track)
276 soup_message_headers_append (msg->request_headers, "DNT", "1");
277
278 /* Process permanent redirects (update feed location) */
279 soup_message_add_status_code_handler (msg, "got_body", 301, (GCallback) network_process_redirect_callback, job);
280 soup_message_add_status_code_handler (msg, "got_body", 308, (GCallback) network_process_redirect_callback, job);
281
282 /* If the feed has "dont use a proxy" selected, use 'session2' which is non-proxy */
283 if (job->request->options && job->request->options->dontUseProxy)
284 soup_session_queue_message (session2, msg, network_process_callback, job);
285 else
286 soup_session_queue_message (session, msg, network_process_callback, job);
287 }
288
289 static void
network_authenticate(SoupSession * session,SoupMessage * msg,SoupAuth * auth,gboolean retrying,gpointer data)290 network_authenticate (
291 SoupSession *session,
292 SoupMessage *msg,
293 SoupAuth *auth,
294 gboolean retrying,
295 gpointer data)
296 {
297 if (!retrying && msg->status_code == SOUP_STATUS_PROXY_UNAUTHORIZED) {
298 soup_auth_authenticate (auth, g_strdup (proxyusername), g_strdup (proxypassword));
299 }
300
301 // FIXME: Handle HTTP 401 too
302 }
303
304 static void
network_set_soup_session_proxy(SoupSession * session,ProxyDetectMode mode,const gchar * host,guint port,const gchar * user,const gchar * password)305 network_set_soup_session_proxy (SoupSession *session, ProxyDetectMode mode, const gchar *host, guint port, const gchar *user, const gchar *password)
306 {
307 SoupURI *uri = NULL;
308
309 switch (mode) {
310 case PROXY_DETECT_MODE_AUTO:
311 /* Sets proxy-resolver to the default resolver, this unsets proxy-uri. */
312 g_object_set (G_OBJECT (session),
313 SOUP_SESSION_PROXY_RESOLVER, g_proxy_resolver_get_default (),
314 NULL);
315 break;
316 case PROXY_DETECT_MODE_NONE:
317 /* Sets proxy-resolver to NULL, this unsets proxy-uri. */
318 g_object_set (G_OBJECT (session),
319 SOUP_SESSION_PROXY_RESOLVER, NULL,
320 NULL);
321 break;
322 case PROXY_DETECT_MODE_MANUAL:
323 uri = soup_uri_new (NULL);
324 soup_uri_set_scheme (uri, SOUP_URI_SCHEME_HTTP);
325 soup_uri_set_host (uri, host);
326 soup_uri_set_port (uri, port);
327 soup_uri_set_user (uri, user);
328 soup_uri_set_password (uri, password);
329 soup_uri_set_path (uri, "/");
330
331 if (SOUP_URI_IS_VALID (uri)) {
332 /* Sets proxy-uri, this unsets proxy-resolver. */
333 g_object_set (G_OBJECT (session),
334 SOUP_SESSION_PROXY_URI, uri,
335 NULL);
336 }
337 soup_uri_free (uri);
338 break;
339 }
340 }
341
342 void
network_init(void)343 network_init (void)
344 {
345 gchar *useragent;
346 SoupCookieJar *cookies;
347 gchar *filename;
348 SoupLogger *logger;
349
350 /* Set an appropriate user agent,
351 * e.g. "Liferea/1.10.0 (Linux; https://lzone.de/liferea/) AppleWebKit (KHTML, like Gecko)" */
352 useragent = g_strdup_printf ("Liferea/%s (%s; %s) AppleWebKit (KHTML, like Gecko)", VERSION, OSNAME, HOMEPAGE);
353
354 /* Session cookies */
355 filename = common_create_config_filename ("session_cookies.txt");
356 cookies = soup_cookie_jar_text_new (filename, TRUE);
357 g_free (filename);
358
359 /* Initialize libsoup */
360 session = soup_session_new_with_options (SOUP_SESSION_USER_AGENT, useragent,
361 SOUP_SESSION_TIMEOUT, 120,
362 SOUP_SESSION_IDLE_TIMEOUT, 30,
363 SOUP_SESSION_ADD_FEATURE, cookies,
364 SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_CONTENT_DECODER,
365 NULL);
366 session2 = soup_session_new_with_options (SOUP_SESSION_USER_AGENT, useragent,
367 SOUP_SESSION_TIMEOUT, 120,
368 SOUP_SESSION_IDLE_TIMEOUT, 30,
369 SOUP_SESSION_ADD_FEATURE, cookies,
370 SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_CONTENT_DECODER,
371 SOUP_SESSION_PROXY_URI, NULL,
372 SOUP_SESSION_PROXY_RESOLVER, NULL,
373 NULL);
374
375 /* Only 'session' gets proxy, 'session2' is for non-proxy requests */
376 network_set_soup_session_proxy (session, network_get_proxy_detect_mode(),
377 network_get_proxy_host (),
378 network_get_proxy_port (),
379 network_get_proxy_username (),
380 network_get_proxy_password ());
381
382 g_signal_connect (session, "authenticate", G_CALLBACK (network_authenticate), NULL);
383
384 /* Soup debugging */
385 if (debug_level & DEBUG_NET) {
386 logger = soup_logger_new (SOUP_LOGGER_LOG_HEADERS, -1);
387 soup_session_add_feature (session, SOUP_SESSION_FEATURE (logger));
388 }
389
390 g_free (useragent);
391 }
392
393 void
network_deinit(void)394 network_deinit (void)
395 {
396 g_free (proxyname);
397 g_free (proxyusername);
398 g_free (proxypassword);
399 }
400
401 ProxyDetectMode
network_get_proxy_detect_mode(void)402 network_get_proxy_detect_mode (void)
403 {
404 return proxymode;
405 }
406
407 const gchar *
network_get_proxy_host(void)408 network_get_proxy_host (void)
409 {
410 return proxyname;
411 }
412
413 guint
network_get_proxy_port(void)414 network_get_proxy_port (void)
415 {
416 return proxyport;
417 }
418
419 const gchar *
network_get_proxy_username(void)420 network_get_proxy_username (void)
421 {
422 return proxyusername;
423 }
424
425 const gchar *
network_get_proxy_password(void)426 network_get_proxy_password (void)
427 {
428 return proxypassword;
429 }
430
431 extern void network_monitor_proxy_changed (void);
432
433 void
network_set_proxy(ProxyDetectMode mode,gchar * host,guint port,gchar * user,gchar * password)434 network_set_proxy (ProxyDetectMode mode, gchar *host, guint port, gchar *user, gchar *password)
435 {
436 g_free (proxyname);
437 g_free (proxyusername);
438 g_free (proxypassword);
439 proxymode = mode;
440 proxyname = host;
441 proxyport = port;
442 proxyusername = user;
443 proxypassword = password;
444
445 /* session will be NULL if we were called from conf_init() as that's called
446 * before net_init() */
447 if (session)
448 network_set_soup_session_proxy (session, mode, host, port, user, password);
449
450 debug4 (DEBUG_NET, "proxy set to http://%s:%s@%s:%d", user, password, host, port);
451
452 network_monitor_proxy_changed ();
453 }
454
455 const char *
network_strerror(gint netstatus,gint httpstatus)456 network_strerror (gint netstatus, gint httpstatus)
457 {
458 const gchar *tmp = NULL;
459 int status = netstatus?netstatus:httpstatus;
460
461 switch (status) {
462 /* Some libsoup transport errors */
463 case SOUP_STATUS_NONE: tmp = _("The update request was cancelled"); break;
464 case SOUP_STATUS_CANT_RESOLVE: tmp = _("Unable to resolve destination host name"); break;
465 case SOUP_STATUS_CANT_RESOLVE_PROXY: tmp = _("Unable to resolve proxy host name"); break;
466 case SOUP_STATUS_CANT_CONNECT: tmp = _("Unable to connect to remote host"); break;
467 case SOUP_STATUS_CANT_CONNECT_PROXY: tmp = _("Unable to connect to proxy"); break;
468 case SOUP_STATUS_SSL_FAILED: tmp = _("A network error occurred, or the other end closed the connection unexpectedly"); break;
469
470 /* http 3xx redirection */
471 case SOUP_STATUS_MOVED_PERMANENTLY: tmp = _("The resource moved permanently to a new location"); break;
472
473 /* http 4xx client error */
474 case SOUP_STATUS_UNAUTHORIZED: tmp = _("You are unauthorized to download this feed. Please update your username and "
475 "password in the feed properties dialog box"); break;
476 case SOUP_STATUS_PAYMENT_REQUIRED: tmp = _("Payment required"); break;
477 case SOUP_STATUS_FORBIDDEN: tmp = _("You're not allowed to access this resource"); break;
478 case SOUP_STATUS_NOT_FOUND: tmp = _("Resource Not Found"); break;
479 case SOUP_STATUS_METHOD_NOT_ALLOWED: tmp = _("Method Not Allowed"); break;
480 case SOUP_STATUS_NOT_ACCEPTABLE: tmp = _("Not Acceptable"); break;
481 case SOUP_STATUS_PROXY_UNAUTHORIZED: tmp = _("Proxy authentication required"); break;
482 case SOUP_STATUS_REQUEST_TIMEOUT: tmp = _("Request timed out"); break;
483 case SOUP_STATUS_GONE: tmp = _("Gone. Resource doesn't exist. Please unsubscribe!"); break;
484 }
485
486 if (!tmp) {
487 if (SOUP_STATUS_IS_TRANSPORT_ERROR (status)) {
488 tmp = _("There was an internal error in the update process");
489 } else if (SOUP_STATUS_IS_REDIRECTION (status)) {
490 tmp = _("Feed not available: Server requested unsupported redirection!");
491 } else if (SOUP_STATUS_IS_CLIENT_ERROR (status)) {
492 tmp = _("Client Error");
493 } else if (SOUP_STATUS_IS_SERVER_ERROR (status)) {
494 tmp = _("Server Error");
495 } else {
496 tmp = _("An unknown networking error happened!");
497 }
498 }
499
500 g_assert (tmp);
501
502 return tmp;
503 }
504
505