mnogosearch-3.4.1/src/htdb.c

/* Copyright (C) 2000-2015 Lavtech.com corp. All rights reserved.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#include "udm_config.h"

#ifdef HAVE_SQL

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <ctype.h>
#include <time.h>

#ifdef WIN32
#include <time.h>
#endif

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#ifdef HAVE_SYS_TIME_H
#include <sys/time.h>
#endif

#include "udm_common.h"
#include "udm_utils.h"
#include "udm_db.h"
#include "udm_url.h"
#include "udm_log.h"
#include "udm_proto.h"
#include "udm_vars.h"
#include "udm_hrefs.h"
#include "udm_db_int.h"
#include "udm_indexer.h"
#include "udm_textlist.h"
#include "udm_parsehtml.h"
#include "udm_parsexml.h"
#include "udm_http.h"

/***********************************************************/
/*  HTDB stuff:  Indexing of database content              */
/***********************************************************/

#define MAXNPART 32


static udm_rc_t
UdmDSTRParseUsingConstStringList(UDM_DSTR *dstr, const char *src,
                                 const char * const *part, size_t nparts)
{
  UdmDSTRReset(dstr);
  for ( ; *src; )
  {
    if (*src == '\\')
    {
      UdmDSTRAppend(dstr, src + 1, 1);
      src+= 2;
      continue;
    }
    if (*src == '$')
    {
      int i= atoi(++src)- 1;

      while(*src >= '0' && *src<= '9')
        src++;
      if (i >= 0 && i < (int) nparts)
      {
        size_t len= strlen(part[i]);
        if (UDM_OK != UdmDSTRAppendURLDecode(dstr, part[i], len))
          return UDM_ERROR;
      }
      continue;
    }
    UdmDSTRAppend(dstr, src++, 1);
  }
  return UDM_OK;
}


static udm_rc_t
include_params(UDM_DB *db, const char *src, char *path,
               UDM_DSTR *dstr, size_t start, int limit)
{
  size_t nparts;
  const char *part[MAXNPART];
  char *lt;

  for (part[nparts= 0]= udm_strtok_r(path, "/", &lt) ;
       part[nparts] && nparts < MAXNPART;
       part[++nparts]= udm_strtok_r(NULL, "/", &lt))
  {
  }

  if (UDM_OK != UdmDSTRParseUsingConstStringList(dstr, src, part, nparts))
    return UDM_ERROR;

  if (limit)
  {
    switch (UdmSQLDBType(db))
    {
      case UDM_DB_MYSQL:
        UdmDSTRAppendf(dstr, " LIMIT %d,%d", (int) start, limit);
        break;
      case UDM_DB_PGSQL:
      default:
        UdmDSTRAppendf(dstr, " LIMIT %d OFFSET %d", limit, (int) start);
        break;
    }
  }
  return UDM_OK;
}


static udm_rc_t
UdmHTDBCreateHTTPResponse(UDM_AGENT *Indexer,
                          UDM_DOCUMENT *Doc,
                          UDM_SQLRES *SQLres)
{
  size_t i;
  for (i= 0; i < UdmSQLNumCols(SQLres); i++)
  {
    size_t len;
    const char *from;
    if (i > 0)
    {
      UdmHTTPBufAppend(&Doc->Buf, "\r\n", 2);
    }
    len= UdmSQLLen(SQLres, 0, i);
    from= UdmSQLValue(SQLres, 0, i);
    if (len == 1 && *from == ' ')
    {
      /*
         Sybase+unixODBC returns space character instead
         of an empty string executing this query:
           SELECT '' FROM t1;
      */
    }
    else
    {
      UdmHTTPBufAppend(&Doc->Buf, from, len);
    }
  }
  return UDM_OK;
}


static void
UdmRemoveWiki(char *str, char *strend)
{
  for ( ; str < strend ; str++)
  {
    if (*str == '[')
    {
      int smcount= 0;
      for (*str++= ' ' ; str < strend ; str++)
      {
        if (*str == ']')
        {
          *str++= ' ';
          break;
        }
        if (*str == '[')
          UdmRemoveWiki(str, strend);
        if (*str == ':')
        {
          *str= ' ';
          smcount++;
        }
        if (smcount < 2)
          *str= ' ';
      }
    }
  }
}


typedef struct udm_htdb_html_helper_st
{
  UDM_DOCUMENT *Doc;
  UDM_CONST_TEXTITEM ConstItem;
  UDM_TEXT_PARAM TextParam;
  const UDM_VAR *Sec;
  UDM_DSTR tbuf;
} UDM_HTDB_HTML_HELPER;


static udm_rc_t
UdmHTDBProcessHTMLText(UDM_HTML_PARSER *parser)
{
  UDM_HTDB_HTML_HELPER *param= (UDM_HTDB_HTML_HELPER *) parser->user_data;

  if (!parser->state.script && !parser->state.comment && !parser->state.style)
  {
    UdmDSTRReset(&param->tbuf);
    if (UdmVarFlags(param->Sec) & UDM_VARFLAG_WIKI)
      UdmRemoveWiki((char*) parser->lasttok.str, (char*) parser->lasttok.str + parser->lasttok.length);
    UdmDSTRAppend(&param->tbuf, parser->lasttok.str, parser->lasttok.length);
    param->ConstItem.text.str= UdmDSTRPtr(&param->tbuf);
    param->ConstItem.text.length= UdmDSTRPtr(&param->tbuf) ? strlen(UdmDSTRPtr(&param->tbuf)) : 0;
    param->TextParam.secno= UdmVarSecno(param->Sec);
    param->ConstItem.section_name.str= UdmVarName(param->Sec);
    param->ConstItem.section_name.length= UdmVarNameLength(param->Sec);
    UdmTextListAddConst(&param->Doc->TextList, &param->ConstItem, &param->TextParam);
  }
  return UDM_OK;
}


static udm_rc_t
UdmHTDBProcessHTML(UDM_HTDB_HTML_HELPER *param, const char *src, size_t srclen)
{
  UDM_HTML_PARSER parser;
  UdmHTMLParserInit(&parser);
  UdmHTMLParserSetUserData(&parser, param);
  UdmHTMLParserSetTextHandler(&parser, UdmHTDBProcessHTMLText);
  return UdmHTMLParserExec(&parser, src, srclen);
}


static udm_rc_t
UdmHTDBProcessNonHTTPResponse(UDM_AGENT *A,
                              UDM_DOCUMENT *Doc,
                              UDM_SQLRES *SQLRes)
{
  UDM_HTDB_HTML_HELPER param;
  udm_rc_t rc= UDM_OK;
  int status= 200;
  size_t row, nrows, ncols= UdmSQLNumCols(SQLRes), length= 0, i;
  char dbuf[UDM_MAXTIMESTRLEN]= "";

  /* Init user data */
  bzero((void*) &param.ConstItem, sizeof(UDM_CONST_TEXTITEM));
  param.Doc= Doc;
  UdmDSTRInit(&param.tbuf, 1024);

  for (row=0, nrows= UdmSQLNumRows(SQLRes); row < nrows; row++)
  {
    size_t col;
    for (col= 0; col  < ncols; col++)
    {
      const char *sqlname= SQLRes->Fields[col].sqlname;
      const char *sqlval= UdmSQLValue(SQLRes, row, col);
      size_t sqlval_length= UdmSQLLen(SQLRes, row, col);
      if ((param.Sec= UdmVarListFind(&param.Doc->Sections, sqlname)))
      {
        param.ConstItem.section_name.str= UdmVarName(param.Sec);
        param.ConstItem.section_name.length= UdmVarNameLength(param.Sec);
        if (UdmVarFlags(param.Sec) & UDM_VARFLAG_HTMLSOURCE)
        {
          /* TODO34: add directly to Doc->Buf ? */
          UdmHTDBProcessHTML(&param, sqlval, sqlval_length);
        }
        else
        {
          param.ConstItem.text.str= sqlval;
          param.ConstItem.text.length= sqlval_length;
          param.TextParam.secno= UdmVarSecno(param.Sec);
          /* TODO34: add directly to Doc->Buf ? */
          UdmTextListAddConst(&param.Doc->TextList, &param.ConstItem, &param.TextParam);
        }
        length+= UdmSQLLen(SQLRes, row, col);
      }
      if (!strcasecmp(sqlname, "status"))
        status= atoi(sqlval);
      else if (!strcasecmp(sqlname, "last_mod_time"))
      {
        int last_mod_time= atoi(sqlval);
        strcpy(dbuf, "Last-Modified: ");
        UdmTime_t2HttpStr(last_mod_time, dbuf + 15, sizeof(dbuf) - 15);
      }
    }
  }

  /* Free user data */
  UdmDSTRFree(&param.tbuf);
  UdmHTTPBufPrintf(&param.Doc->Buf,
                   "HTTP/1.0 %d %s\r\n"
                   /*"HTDB-Content-Length: %d\r\n"*/
                   "Content-Type: mnogosearch/htdb; charset=%s\r\n%s%s\r\n",
                   status, UdmHTTPErrMsg(status),
                   /*(int) length,*/
                   A->Conf->lcs->name,
                   dbuf[0] ? dbuf : "", dbuf[0] ? "\r\n" : "");


  UdmHTTPBufAppendf(&param.Doc->Buf, "<doc>\n");
  for (i= 0; i < param.Doc->TextList.nitems; i++)
  {
    UDM_TEXTITEM *Item= &param.Doc->TextList.Item[i];
    UdmHTTPBufAppendf(&param.Doc->Buf,
                      "<sec name=\"%s\"><![CDATA[%s]]></sec>\n",
                      Item->section_name, Item->str);
  }
  UdmHTTPBufAppendf(&param.Doc->Buf, "</doc>\n");
  UdmTextListFree(&param.Doc->TextList);
  return rc;
}


static udm_rc_t
UdmHTDBGetDocument(UDM_AGENT *Indexer,
                   UDM_DOCUMENT *Doc,
                   UDM_DB *db,
                   const UDM_URL *realURL)
{
  udm_rc_t rc= UDM_OK;
  const char  *htdbdoc= UdmVarListFindStr(&Doc->Sections, "HTDBDoc", "");
  char real_path[1024]= "";
  UDM_SQLRES SQLres;
  UDM_DSTR qbuf;

  UdmDSTRInit(&qbuf, 128);
  udm_snprintf(real_path, sizeof(real_path) - 1, "%s%s",
               realURL->path, realURL->filename);
  include_params(db, htdbdoc, real_path, &qbuf, 0, 0);
  UdmLog(Indexer, UDM_LOG_DEBUG, "HTDBDoc: %s", UdmDSTRPtr(&qbuf));
  if (UDM_OK != (rc= UdmDBSQLQuery(Indexer, db, &SQLres, UdmDSTRPtr(&qbuf))))
    goto ret;

  if (UdmSQLNumRows(&SQLres) == 1)
  {
    /*
      Can't use strncmp() in combination with UDM_CSTR_WITH_LEN.
      It fails with -O2 on Linux, because strncmp() is defined
      as an optimized macros in /usr/include/bits/string2.h
    */
    if (!strncmp(UdmSQLValue(&SQLres, 0, 0), "HTTP/", 5))
      UdmHTDBCreateHTTPResponse(Indexer, Doc, &SQLres);
    else
      UdmHTDBProcessNonHTTPResponse(Indexer, Doc, &SQLres);
  }
  else
  {
    UdmHTTPBufAppendf(&Doc->Buf, "HTTP/1.0 404 Not Found\r\n\r\n");
  }
  UdmSQLFree(&SQLres);
ret:
  UdmDSTRFree(&qbuf);
  return rc;
}


static udm_rc_t
UdmHTDBGetDirectoryList(UDM_AGENT *Indexer,
                        UDM_DOCUMENT *Doc,
                        UDM_DB *db,
                        const UDM_URL *realURL)
{
  udm_rc_t rc= UDM_OK;
  size_t  i, start;
  urlid_t  url_id= UdmVarListFindInt(&Doc->Sections, "ID", 0);
  const char *htdblist= UdmVarListFindStr(&Doc->Sections,"HTDBList","");
  int htdblimit= UdmVarListFindInt(&Doc->Sections, "HTDBLimit", 0);
  udm_bool_t usehtdburlid = UdmVarListFindBool(&Indexer->Conf->Vars, "UseHTDBURLId", UDM_FALSE);
  int done, hops= UdmVarListFindInt(&Doc->Sections,"Hops",0);
  UDM_DSTR qbuf;

  UdmDSTRInit(&qbuf, 128);
  UdmHTTPBufAppendf(&Doc->Buf,
                    "HTTP/1.0 200 OK\r\nContent-type: text/html\r\n\r\n"
                    "<HTML><BODY>\n"
                    "</BODY></HTML>\n");

  for (start=0, done= 0; !done; )
  {
    size_t nrows;
    char real_path[1024]="";
    UDM_SQLRES SQLres;

    udm_snprintf(real_path, sizeof(real_path), "%s", realURL->path);
    include_params(db, htdblist, real_path, &qbuf, start, htdblimit);
    UdmLog(Indexer, UDM_LOG_DEBUG, "HTDBList: %s", UdmDSTRPtr(&qbuf));
    if (UDM_OK != (rc= UdmDBSQLQuery(Indexer, db, &SQLres, UdmDSTRPtr(&qbuf))))
      goto ret;

    nrows= UdmSQLNumRows(&SQLres);
    done= htdblimit ? (htdblimit != (int) nrows) : 1;
    start+= nrows;

    for (i= 0; i < nrows; i++)
    {
      UDM_HREFPARAM HrefParam;
      UDM_CONST_STR url;
      UdmSQLValueToConstStr(&url, &SQLres, i, 0);
      UdmHrefParamInit(&HrefParam);
      HrefParam.referrer= url_id;
      HrefParam.hops= hops+1;
      HrefParam.rec_id= usehtdburlid ? atoi(url.str) : 0;
      HrefParam.link_source= UDM_LINK_SOURCE_HTDB;
      UdmHrefListAddConstStr(&Doc->Hrefs, &HrefParam, &url);
    }
    UdmSQLFree(&SQLres);
    UdmDocStoreHrefs(Indexer, Doc);
    UdmHrefListFree(&Doc->Hrefs);
    UdmStoreHrefs(Indexer);
  }
ret:
  UdmDSTRFree(&qbuf);
  return rc;
}


udm_rc_t
UdmHTDBGet(UDM_AGENT *Indexer,UDM_DOCUMENT *Doc)
{
  UDM_URL     realURL;
  UDM_DB      dbnew, *db= NULL;
  const char  *url=UdmVarListFindStr(&Doc->Sections,"URL","");
  const char  *htdbaddr = UdmVarListFindStr(&Doc->Sections, "HTDBAddr", NULL);
  udm_rc_t rc= UDM_OK;

  UdmHTTPBufReset(&Doc->Buf);
  UdmURLInit(&realURL);
  UdmURLParse(&realURL, url);

  if (htdbaddr)
  {
    UdmDBInit(&dbnew);
    db= &dbnew;
    if (UDM_OK != (rc= UdmDBSetAddr(db, htdbaddr)))
    {
      if (rc == UDM_NOTARGET)
      {
        UdmLog(Indexer, UDM_LOG_ERROR, "Unsupported DBAddr");
        rc= UDM_ERROR;
      }
      else
      {
        UdmLog(Indexer, UDM_LOG_ERROR, "UdmDBSetAddr failed");
        rc= UDM_ERROR;
      }
      goto HTDBexit;
    }
  }
  else
  {
    if (Indexer->Conf->DBList.nitems != 1)
    {
      UdmLog(Indexer, UDM_LOG_ERROR, "HTDB cannot work with multiple DBAddr without HTDBAddr");
      rc= UDM_ERROR;
      goto HTDBexit;
    }
    db= &Indexer->Conf->DBList.Item[0];
  }

  rc= realURL.filename != NULL ?
      UdmHTDBGetDocument(Indexer, Doc, db, &realURL) :
      UdmHTDBGetDirectoryList(Indexer, Doc, db, &realURL);

HTDBexit:
  if (db == &dbnew)
    UdmDBFree(db);
  UdmURLFree(&realURL);
  return rc;
}


typedef struct
{
  UDM_DOCUMENT *Doc;
  UDM_CONST_STR secname;
} XML_PARSER_DATA;


static udm_rc_t
startElement(UDM_XML_PARSER *parser, const char *str, size_t len)
{
  return UDM_OK;
}


static udm_rc_t
endElement(UDM_XML_PARSER *parser, const char *str, size_t len)
{
  XML_PARSER_DATA *D= (XML_PARSER_DATA*) parser->user_data;
  if (!udm_strnncasecmp(str, len, UDM_CSTR_WITH_LEN("/doc/sec")))
    UdmConstStrSet(&D->secname, NULL, 0);
  return UDM_OK;
}


static udm_rc_t
Text(UDM_XML_PARSER *parser, const char *str, size_t len)
{
  XML_PARSER_DATA *D= (XML_PARSER_DATA*) parser->user_data;
  if (!strcasecmp(parser->attr, "/doc/sec@name"))
  {
    UdmConstStrSet(&D->secname, str, len);
  }
  else if (!strcasecmp(parser->attr, "/doc/sec"))
  {
    if (D->secname.length)
    {
      char secname[128];
      UDM_CONST_TEXTITEM ConstItem;
      const UDM_VAR *Sec;
      UdmConstTextItemInit(&ConstItem);
      udm_snprintf(secname, sizeof(secname), "%.*s",
                   (int) D->secname.length, D->secname.str);
      if ((Sec= UdmVarListFind(&D->Doc->Sections, secname)))
      {
        UDM_TEXT_PARAM TextParam;
        ConstItem.section_name.str= secname;
        ConstItem.section_name.length= strlen(secname);
        ConstItem.text.str= str;
        ConstItem.text.length= len;
        UdmTextParamInit(&TextParam, UDM_TEXTLIST_FLAG_NONE, UdmVarSecno(Sec));
        UdmTextListAddConst(&D->Doc->TextList, &ConstItem, &TextParam);
      }
      /*
      printf("Sec[%d] '%.*s'='%.*s'\n",
             Sec ? Sec->section : 0,
             (int) D->secname.length, D->secname.str,
             (int) len, str);
      */
    }
  }
  return UDM_OK;
}


static udm_rc_t
UdmHTDBParseInternal(UDM_AGENT *Indexer, UDM_DOCUMENT *Doc,
                     const UDM_CONST_STR *content)
{
  udm_rc_t rc= UDM_OK;
  XML_PARSER_DATA Data;
  UDM_XML_PARSER parser;

  UdmXMLParserCreate(&parser);
  bzero(&Data, sizeof(Data));
  Data.Doc= Doc;

  UdmXMLSetUserData(&parser, &Data);
  UdmXMLSetEnterHandler(&parser, startElement);
  UdmXMLSetLeaveHandler(&parser, endElement);
  UdmXMLSetValueHandler(&parser, Text);

  if (UDM_OK!= (rc= UdmXMLParserExec(&parser, content->str, content->length)))
  {
    char err[256];
    udm_snprintf(err, sizeof(err),
                 "XML parsing error: %s at line %d pos %d",
                  UdmXMLErrorString(&parser),
                  (int) UdmXMLErrorLineno(&parser),
                  (int) UdmXMLErrorPos(&parser));
    UdmVarListReplaceStr(&Doc->Sections, "X-Reason", err);
  }

  UdmXMLParserFree(&parser);
  return rc;
}


udm_rc_t
UdmHTDBParse(UDM_AGENT *Indexer, UDM_DOCUMENT *Doc)
{
  UDM_CONST_STR content;

  if (UdmHTTPBufContentToConstStr(&Doc->Buf, &content))
    return UDM_ERROR;

  return UdmHTDBParseInternal(Indexer, Doc, &content);
}


size_t
UdmHTDBExcerptSource(UDM_AGENT *Agent,
                     UDM_QUERY *Query,
                     UDM_DOCUMENT *Doc,
                     const UDM_CONST_STR *content,
                     UDM_DSTR *dstr)
{
  size_t i;

  UdmVarListAddStr(&Doc->Sections, "body", "");

  if (UDM_OK != UdmHTDBParseInternal(Agent, Doc, content))
    return 0;

  /* Collect body chunks from Doc->TextList into dstr */
  for (i= 0; i < Doc->TextList.nitems; i++)
  {
    UDM_TEXTITEM *Item= &Doc->TextList.Item[i];
    if (!strcmp(Item->section_name, "body"))
    {
      if (UdmDSTRLength(dstr))
        UdmDSTRAppend(dstr, " ", 1);
      UdmDSTRAppend(dstr, Item->str, strlen(Item->str));
    }
  }
  return UdmDSTRLength(dstr);
}


#endif /* HAVE_SQL */