dillo-3.0.5/src/url.c

/*
 * File: url.c
 *
 * Copyright (C) 2001-2009 Jorge Arellano Cid <jcid@dillo.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 */

/*
 * Parse and normalize all URL's inside Dillo.
 *  - <scheme> <authority> <path> <query> and <fragment> point to 'buffer'.
 *  - 'url_string' is built upon demand (transparent to the caller).
 *  - 'hostname' and 'port' are also being handled on demand.
 */

/*
 * Regular Expression as given in RFC3986 for URL parsing.
 *
 *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
 *   12            3  4          5       6  7        8 9
 *
 *  scheme    = $2
 *  authority = $4
 *  path      = $5
 *  query     = $7
 *  fragment  = $9
 *
 *
 *  RFC-2396 BNF:
 *
 *  absoluteURI = scheme ":" (hier_part | opaque_part)
 *  hier_part   = (net_path | abs_path) ["?" query]
 *  net_path    = "//" authority[abs_path]
 *  abs_path    = "/" path_segments
 *
 *  Notes:
 *    - "undefined" means "preceeding separator does not appear".
 *    - path is never "undefined" though it may be "empty".
 */

#include <stdlib.h>
#include <string.h>
#include <ctype.h>

#include "url.h"
#include "msg.h"

static const char *HEX = "0123456789ABCDEF";

/* URL-field compare methods */
#define URL_STR_FIELD_CMP(s1,s2) \
   (s1) && (s2) ? strcmp(s1,s2) : !(s1) && !(s2) ? 0 : (s1) ? 1 : -1
#define URL_STR_FIELD_I_CMP(s1,s2) \
   (s1) && (s2) ? dStrAsciiCasecmp(s1,s2) : !(s1) && !(s2) ? 0 : (s1) ? 1 : -1

/*
 * Return the url as a string.
 * (initializing 'url_string' field if necessary)
 */
char *a_Url_str(const DilloUrl *u)
{
   /* Internal url handling IS transparent to the caller */
   DilloUrl *url = (DilloUrl *) u;

   dReturn_val_if_fail (url != NULL, NULL);

   if (!url->url_string) {
      url->url_string = dStr_sized_new(60);
      dStr_sprintf(
         url->url_string, "%s%s%s%s%s%s%s%s%s%s",
         url->scheme    ? url->scheme : "",
         url->scheme    ? ":" : "",
         url->authority ? "//" : "",
         url->authority ? url->authority : "",
         // (url->path && url->path[0] != '/' && url->authority) ? "/" : "",
         (url->authority && (!url->path || *url->path != '/')) ? "/" : "",
         url->path      ? url->path : "",
         url->query     ? "?" : "",
         url->query     ? url->query : "",
         url->fragment  ? "#" : "",
         url->fragment  ? url->fragment : "");
   }

   return url->url_string->str;
}

/*
 * Return the hostname as a string.
 * (initializing 'hostname' and 'port' fields if necessary)
 * Note: a similar approach can be taken for user:password auth.
 */
const char *a_Url_hostname(const DilloUrl *u)
{
   char *p;
   /* Internal url handling IS transparent to the caller */
   DilloUrl *url = (DilloUrl *) u;

   if (!url->hostname && url->authority) {
      if (url->authority[0] == '[' && (p = strchr(url->authority, ']'))) {
         /* numeric ipv6 address, strip the brackets */
         url->hostname = dStrndup(url->authority + 1,
                                  (uint_t)(p - url->authority - 1));
         if ((p = strchr(p, ':'))) {
            url->port = strtol(p + 1, NULL, 10);
         }
      } else {
         /* numeric ipv4 or hostname */
         if ((p = strchr(url->authority, ':'))) {
            url->port = strtol(p + 1, NULL, 10);
            url->hostname = dStrndup(url->authority,
                                     (uint_t)(p - url->authority));
         } else {
            url->hostname = url->authority;
         }
      }
   }

   return url->hostname;
}

/*
 *  Create a DilloUrl object and initialize it.
 *  (buffer, scheme, authority, path, query and fragment).
 */
static DilloUrl *Url_object_new(const char *uri_str)
{
   DilloUrl *url;
   char *s, *p;

   dReturn_val_if_fail (uri_str != NULL, NULL);

   url = dNew0(DilloUrl, 1);

   /* remove leading & trailing space from buffer */
   url->buffer = dStrstrip(dStrdup(uri_str));

   s = (char *) url->buffer;
   p = strpbrk(s, ":/?#");
   if (p && p[0] == ':' && p > s) {                /* scheme */
      *p = 0;
      url->scheme = s;
      s = ++p;
   }
   /* p = strpbrk(s, "/"); */
   if (p == s && p[0] == '/' && p[1] == '/') {     /* authority */
      s = p + 2;
      p = strpbrk(s, "/?#");
      if (p) {
         memmove(s - 2, s, (size_t)MAX(p - s, 1));
         url->authority = s - 2;
         p[-2] = 0;
         s = p;
      } else if (*s) {
         url->authority = s;
         return url;
      }
   }

   p = strpbrk(s, "?#");
   if (p) {                                        /* path */
      url->path = (p > s) ? s : NULL;
      s = p;
   } else if (*s) {
      url->path = s;
      return url;
   }

   p = strpbrk(s, "?#");
   if (p && p[0] == '?') {                         /* query */
      *p = 0;
      s = p + 1;
      url->query = s;
      p = strpbrk(s, "#");
      url->flags |= URL_Get;
   }
   if (p && p[0] == '#') {                         /* fragment */
      *p = 0;
      s = p + 1;
      url->fragment = s;
   }

   return url;
}

/*
 *  Free a DilloUrl
 *  Do nothing if the argument is NULL
 */
void a_Url_free(DilloUrl *url)
{
   if (url) {
      if (url->url_string)
         dStr_free(url->url_string, TRUE);
      if (url->hostname != url->authority)
         dFree((char *)url->hostname);
      dFree((char *)url->buffer);
      dStr_free(url->data, 1);
      dFree((char *)url->alt);
      dFree(url);
   }
}

/*
 * Resolve the URL as RFC3986 suggests.
 */
static Dstr *Url_resolve_relative(const char *RelStr,
                                  DilloUrl *BaseUrlPar,
                                  const char *BaseStr)
{
   char *p, *s, *e;
   int i;
   Dstr *SolvedUrl, *Path;
   DilloUrl *RelUrl, *BaseUrl = NULL;

   /* parse relative URL */
   RelUrl = Url_object_new(RelStr);

   if (BaseUrlPar) {
      BaseUrl = BaseUrlPar;
   } else if (RelUrl->scheme == NULL) {
      /* only required when there's no <scheme> in RelStr */
      BaseUrl = Url_object_new(BaseStr);
   }

   SolvedUrl = dStr_sized_new(64);
   Path = dStr_sized_new(64);

   /* path empty && scheme and authority undefined */
   if (!RelUrl->path && !RelUrl->scheme && !RelUrl->authority) {
      dStr_append(SolvedUrl, BaseStr);
      if ((p = strchr(SolvedUrl->str, '#')))
         dStr_truncate(SolvedUrl, p - SolvedUrl->str);
      if (!BaseUrl->path)
         dStr_append_c(SolvedUrl, '/');

      if (RelUrl->query) {                        /* query */
         if (BaseUrl->query)
            dStr_truncate(SolvedUrl, BaseUrl->query - BaseUrl->buffer - 1);
         dStr_append_c(SolvedUrl, '?');
         dStr_append(SolvedUrl, RelUrl->query);
      }
      if (RelUrl->fragment) {                    /* fragment */
         dStr_append_c(SolvedUrl, '#');
         dStr_append(SolvedUrl, RelUrl->fragment);
      }
      goto done;

   } else if (RelUrl->scheme) {                  /* scheme */
      dStr_append(SolvedUrl, RelStr);
      goto done;

   } else if (RelUrl->authority) {               /* authority */
      // Set the Path buffer and goto "STEP 7";
      if (RelUrl->path)
         dStr_append(Path, RelUrl->path);

   } else {
      if (RelUrl->path && RelUrl->path[0] == '/') {   /* absolute path */
         ; /* Ignore BaseUrl path */
      } else if (BaseUrl->path) {                     /* relative path */
         dStr_append(Path, BaseUrl->path);
         for (i = Path->len; --i >= 0 && Path->str[i] != '/'; ) ;
         if (i >= 0 && Path->str[i] == '/')
            dStr_truncate(Path, ++i);
      }
      if (RelUrl->path)
         dStr_append(Path, RelUrl->path);

      // erase "./"
      while ((p=strstr(Path->str, "./")) &&
             (p == Path->str || p[-1] == '/'))
         dStr_erase(Path, p - Path->str, 2);
      // erase last "."
      if (Path->len && Path->str[Path->len - 1] == '.' &&
          (Path->len == 1 || Path->str[Path->len - 2] == '/'))
         dStr_truncate(Path, Path->len - 1);

      // erase "<segment>/../" and "<segment>/.."
      s = p = Path->str;
      while ( (p = strstr(p, "/..")) != NULL ) {
         if (p[3] == '/' || !p[3]) { //  "/../" | "/.."
            for (e = p + 3 ; p > s && p[-1] != '/'; --p) ;
            dStr_erase(Path, p - Path->str, e - p + (p > s && *e != 0));
            p -= (p > Path->str);
         } else
            p += 3;
      }
   }

   /* STEP 7
    */

   /* scheme */
   if (BaseUrl->scheme) {
      dStr_append(SolvedUrl, BaseUrl->scheme);
      dStr_append_c(SolvedUrl, ':');
   }

   /* authority */
   if (RelUrl->authority) {
      dStr_append(SolvedUrl, "//");
      dStr_append(SolvedUrl, RelUrl->authority);
   } else if (BaseUrl->authority) {
      dStr_append(SolvedUrl, "//");
      dStr_append(SolvedUrl, BaseUrl->authority);
   }

   /* path */
   if ((RelUrl->authority || BaseUrl->authority) &&
       ((Path->len == 0 && (RelUrl->query || RelUrl->fragment)) ||
        (Path->len && Path->str[0] != '/')))
      dStr_append_c(SolvedUrl, '/'); /* hack? */
   dStr_append(SolvedUrl, Path->str);

   /* query */
   if (RelUrl->query) {
      dStr_append_c(SolvedUrl, '?');
      dStr_append(SolvedUrl, RelUrl->query);
   }

   /* fragment */
   if (RelUrl->fragment) {
      dStr_append_c(SolvedUrl, '#');
      dStr_append(SolvedUrl, RelUrl->fragment);
   }

done:
   dStr_free(Path, TRUE);
   a_Url_free(RelUrl);
   if (BaseUrl != BaseUrlPar)
      a_Url_free(BaseUrl);
   return SolvedUrl;
}

/*
 *  Transform (and resolve) an URL string into the respective DilloURL.
 *  If URL  =  "http://dillo.sf.net:8080/index.html?long#part2"
 *  then the resulting DilloURL should be:
 *  DilloURL = {
 *     url_string         = "http://dillo.sf.net:8080/index.html?long#part2"
 *     scheme             = "http"
 *     authority          = "dillo.sf.net:8080:
 *     path               = "/index.html"
 *     query              = "long"
 *     fragment           = "part2"
 *     hostname           = "dillo.sf.net"
 *     port               = 8080
 *     flags              = URL_Get
 *     data               = Dstr * ("")
 *     alt                = NULL
 *     ismap_url_len      = 0
 *  }
 *
 *  Return NULL if URL is badly formed.
 */
DilloUrl* a_Url_new(const char *url_str, const char *base_url)
{
   DilloUrl *url;
   char *urlstr = (char *)url_str;  /* auxiliar variable, don't free */
   char *p, *str1 = NULL, *str2 = NULL;
   Dstr *SolvedUrl;
   int i, n_ic, n_ic_spc;

   dReturn_val_if_fail (url_str != NULL, NULL);

   /* Count illegal characters (0x00-0x1F, 0x7F-0xFF and space) */
   n_ic = n_ic_spc = 0;
   for (p = (char*)url_str; *p; p++) {
      n_ic_spc += (*p == ' ') ? 1 : 0;
      n_ic += (*p != ' ' && *p > 0x1F && *p < 0x7F) ? 0 : 1;
   }
   if (n_ic) {
      /* Encode illegal characters (they could also be stripped).
       * There's no standard for illegal chars; we chose to encode. */
      p = str1 = dNew(char, strlen(url_str) + 2*n_ic + 1);
      for (i = 0; url_str[i]; ++i)
         if (url_str[i] > 0x1F && url_str[i] < 0x7F && url_str[i] != ' ')
            *p++ = url_str[i];
         else  {
           *p++ = '%';
           *p++ = HEX[(url_str[i] >> 4) & 15];
           *p++ = HEX[url_str[i] & 15];
         }
      *p = 0;
      urlstr = str1;
   }

   /* let's use a heuristic to set http: as default */
   if (!base_url) {
      base_url = "http:";
      if (urlstr[0] != '/') {
         p = strpbrk(urlstr, "/#?:");
         if (!p || *p != ':')
            urlstr = str2 = dStrconcat("//", urlstr, NULL);
      } else if (urlstr[1] != '/')
         urlstr = str2 = dStrconcat("/", urlstr, NULL);
   }

   /* Resolve the URL */
   SolvedUrl = Url_resolve_relative(urlstr, NULL, base_url);
   _MSG("SolvedUrl = %s\n", SolvedUrl->str);

   /* Fill url data */
   url = Url_object_new(SolvedUrl->str);
   url->data = dStr_new("");
   url->url_string = SolvedUrl;
   url->illegal_chars = n_ic;
   url->illegal_chars_spc = n_ic_spc;

   dFree(str1);
   dFree(str2);
   return url;
}


/*
 *  Duplicate a Url structure
 */
DilloUrl* a_Url_dup(const DilloUrl *ori)
{
   DilloUrl *url;

   url = Url_object_new(URL_STR_(ori));
   dReturn_val_if_fail (url != NULL, NULL);

   url->url_string           = dStr_new(URL_STR(ori));
   url->port                 = ori->port;
   url->flags                = ori->flags;
   url->alt                  = dStrdup(ori->alt);
   url->ismap_url_len        = ori->ismap_url_len;
   url->illegal_chars        = ori->illegal_chars;
   url->illegal_chars_spc    = ori->illegal_chars_spc;
   url->data                 = dStr_sized_new(URL_DATA(ori)->len);
   dStr_append_l(url->data, URL_DATA(ori)->str, URL_DATA(ori)->len);
   return url;
}

/*
 *  Compare two Url's to check if they're the same, or which one is bigger.
 *
 *  The fields which are compared here are:
 *  <scheme>, <authority>, <path>, <query> and <data>
 *  Other fields are left for the caller to check
 *
 *  Return value: 0 if equal, > 0 if A > B, < 0 if A < B.
 *
 *  Note: this function defines a sorting order different from strcmp!
 */
int a_Url_cmp(const DilloUrl *A, const DilloUrl *B)
{
   int st;

   dReturn_val_if_fail(A && B, 1);

   if (A == B ||
       ((st = URL_STR_FIELD_I_CMP(A->authority, B->authority)) == 0 &&
        (st = strcmp(A->path ? A->path + (*A->path == '/') : "",
                     B->path ? B->path + (*B->path == '/') : "")) == 0 &&
        //(st = URL_STR_FIELD_CMP(A->path, B->path)) == 0 &&
        (st = URL_STR_FIELD_CMP(A->query, B->query)) == 0 &&
        (st = dStr_cmp(A->data, B->data)) == 0 &&
        (st = URL_STR_FIELD_I_CMP(A->scheme, B->scheme)) == 0))
      return 0;
   return st;
}

/*
 * Set DilloUrl flags
 */
void a_Url_set_flags(DilloUrl *u, int flags)
{
   if (u)
      u->flags = flags;
}

/*
 * Set DilloUrl data (like POST info, etc.)
 */
void a_Url_set_data(DilloUrl *u, Dstr **data)
{
   if (u) {
      dStr_free(u->data, 1);
      u->data = *data;
      *data = NULL;
   }
}

/*
 * Set DilloUrl alt (alternate text to the URL. Used by image maps)
 */
void a_Url_set_alt(DilloUrl *u, const char *alt)
{
   if (u) {
      dFree((char *)u->alt);
      u->alt = dStrdup(alt);
   }
}

/*
 * Set DilloUrl ismap coordinates
 * (this is optimized for not hogging the CPU)
 */
void a_Url_set_ismap_coords(DilloUrl *u, char *coord_str)
{
   dReturn_if_fail (u && coord_str);

   if (!u->ismap_url_len) {
      /* Save base-url length (without coords) */
      u->ismap_url_len  = URL_STR_(u) ? u->url_string->len : 0;
      a_Url_set_flags(u, URL_FLAGS(u) | URL_Ismap);
   }
   if (u->url_string) {
      dStr_truncate(u->url_string, u->ismap_url_len);
      dStr_append(u->url_string, coord_str);
      u->query = u->url_string->str + u->ismap_url_len + 1;
   }
}

/*
 * Given an hex octet (e.g., e3, 2F, 20), return the corresponding
 * character if the octet is valid, and -1 otherwise
 */
static int Url_decode_hex_octet(const char *s)
{
   int hex_value;
   char *tail, hex[3];

   if (s && (hex[0] = s[0]) && (hex[1] = s[1])) {
      hex[2] = 0;
      hex_value = strtol(hex, &tail, 16);
      if (tail - hex == 2)
        return hex_value;
   }
   return -1;
}

/*
 * Parse possible hexadecimal octets in the URI path.
 * Returns a new allocated string.
 */
char *a_Url_decode_hex_str(const char *str)
{
   char *new_str, *dest;
   int i, val;

   if (!str)
      return NULL;

   /* most cases won't have hex octets */
   if (!strchr(str, '%'))
      return dStrdup(str);

   dest = new_str = dNew(char, strlen(str) + 1);

   for (i = 0; str[i]; i++) {
      *dest++ = (str[i] == '%' && (val = Url_decode_hex_octet(str+i+1)) >= 0) ?
                i+=2, val : str[i];
   }
   *dest++ = 0;

   new_str = dRealloc(new_str, sizeof(char) * (dest - new_str));
   return new_str;
}

/*
 * Urlencode 'str'
 * -RL :: According to the RFC 1738, only alphanumerics, the special
 *        characters "$-_.+!*'(),", and reserved characters ";/?:@=&" used
 *        for their *reserved purposes* may be used unencoded within a URL.
 * We'll escape everything but alphanumeric and "-_.*" (as lynx).  --Jcid
 *
 * Note: the content type "application/x-www-form-urlencoded" is used:
 *       i.e., ' ' -> '+' and '\n' -> CR LF (see HTML 4.01, Sec. 17.13.4)
 */
char *a_Url_encode_hex_str(const char *str)
{
   static const char *const verbatim = "-_.*";
   char *newstr, *c;

   if (!str)
      return NULL;

   newstr = dNew(char, 6*strlen(str)+1);

   for (c = newstr; *str; str++)
      if ((dIsalnum(*str) && isascii(*str)) || strchr(verbatim, *str))
         *c++ = *str;
      else if (*str == ' ')
         *c++ = '+';
      else if (*str == '\n') {
         *c++ = '%';
         *c++ = '0';
         *c++ = 'D';
         *c++ = '%';
         *c++ = '0';
         *c++ = 'A';
      } else {
         *c++ = '%';
         *c++ = HEX[(*str >> 4) & 15];
         *c++ = HEX[*str & 15];
      }
   *c = 0;

  return newstr;
}


/*
 * RFC-3986 suggests this stripping when "importing" URLs from other media.
 * Strip: "URL:", enclosing < >, and embedded whitespace.
 * (We also strip illegal chars: 00-1F and 7F-FF)
 */
char *a_Url_string_strip_delimiters(const char *str)
{
   char *p, *new_str, *text;

   new_str = text = dStrdup(str);

   if (new_str) {
      if (strncmp(new_str, "URL:", 4) == 0)
         text += 4;
      if (*text == '<')
         text++;

      for (p = new_str; *text; text++)
         if (*text > 0x1F && *text < 0x7F && *text != ' ')
            *p++ = *text;
      if (p > new_str && p[-1] == '>')
         --p;
      *p = 0;
   }
   return new_str;
}

/*
 * Is the provided hostname an IP address?
 */
static bool_t Url_host_is_ip(const char *host)
{
   uint_t len;

   if (!host || !*host)
      return FALSE;

   len = strlen(host);

   if (len == strspn(host, "0123456789.")) {
      _MSG("an IPv4 address\n");
      return TRUE;
   }
   if (strchr(host, ':') &&
       (len == strspn(host, "0123456789abcdefABCDEF:."))) {
      /* The precise format is shown in section 3.2.2 of rfc 3986 */
      MSG("an IPv6 address\n");
      return TRUE;
   }
   return FALSE;
}

/*
 * How many internal dots are in the public portion of this hostname?
 * e.g., for "www.dillo.org", it is one because everything under "dillo.org",
 * as a .org domain, is part of one organization.
 *
 * Of course this is only a simple and imperfect approximation of
 * organizational boundaries.
 */
static uint_t Url_host_public_internal_dots(const char *host)
{
   uint_t ret = 1;

   if (host) {
      int start, after, tld_len;

      /* We may be able to trust the format of the host string more than
       * I am here. Trailing dots and no dots are real possibilities, though.
       */
      after = strlen(host);
      if (after > 0 && host[after - 1] == '.')
         after--;
      start = after;
      while (start > 0 && host[start - 1] != '.')
         start--;
      tld_len = after - start;

      if (tld_len > 0) {
         /* These TLDs were chosen by examining the current publicsuffix list
          * in October 2014 and picking out those where it was simplest for
          * them to describe the situation by beginning with a "*.[tld]" rule
          * or every rule was "[something].[tld]".
          *
          * TODO: Consider the old publicsuffix code again. This TLD list has
          * shrunk and shrunk over the years, and has become a poorer and
          * poorer approximation of administrative boundaries.
          */
         const char *const tlds[] = {"bd","bn","ck","cy","er","fj","fk",
                                     "gu","il","jm","ke","kh","kw","mm","mz",
                                     "ni","np","pg","ye","za","zm","zw"};
         uint_t i, tld_num = sizeof(tlds) / sizeof(tlds[0]);

         for (i = 0; i < tld_num; i++) {
            if (strlen(tlds[i]) == (uint_t) tld_len &&
                !dStrnAsciiCasecmp(tlds[i], host + start, tld_len)) {
               _MSG("TLD code matched %s\n", tlds[i]);
               ret++;
               break;
            }
         }
      }
   }
   return ret;
}

/*
 * Given a URL host string, return the portion that is public, i.e., the
 * domain that is in a registry outside the organization.
 * For 'www.dillo.org', that would be 'dillo.org'.
 */
static const char *Url_host_find_public_suffix(const char *host)
{
   const char *s;
   uint_t dots;

   if (!host || !*host || Url_host_is_ip(host))
      return host;

   s = host;

   while (s[1])
      s++;

   if (s > host && *s == '.') {
      /* don't want to deal with trailing dot */
      s--;
   }

   dots = Url_host_public_internal_dots(host);

   /* With a proper host string, we should not be pointing to a dot now. */

   while (s > host) {
      if (s[-1] == '.') {
         if (dots == 0)
            break;
         else
            dots--;
      }
      s--;
   }

   _MSG("public suffix of %s is %s\n", host, s);
   return s;
}

bool_t a_Url_same_organization(const DilloUrl *u1, const DilloUrl *u2)
{
   if (!u1 || !u2)
      return FALSE;

   return dStrAsciiCasecmp(Url_host_find_public_suffix(URL_HOST(u1)),
                           Url_host_find_public_suffix(URL_HOST(u2)))
          ? FALSE : TRUE;
}