mnogosearch-3.4.1/src/popularity.c

/* Copyright (C) 2000-2015 Lavtech.com corp. All rights reserved.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#include "udm_config.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>

#include "udm_common.h"
#include "udm_utils.h"
#include "udm_log.h"
#include "udm_searchtool.h"
#include "udm_coords.h"
#include "udm_db.h"


typedef struct
{
  urlid_t id;
  int popularity;
} UDM_URLIPOP;


typedef struct
{
  size_t nitems;
  size_t mitems;
  UDM_URLIPOP *Item;
} UDM_URLIPOPLIST;


static void
UdmURLIPopListInit(UDM_URLIPOPLIST *List)
{
  bzero((void*) List, sizeof(*List));
}


static void
UdmURLIPopListFree(UDM_URLIPOPLIST *List)
{
  UdmFree(List->Item);
}


static udm_rc_t
UdmURLIPopListAlloc(UDM_URLIPOPLIST *List, size_t mitems)
{
  if (!(List->Item= (UDM_URLIPOP *) UdmMalloc(mitems * sizeof(UDM_URLIPOP))))
    return UDM_ERROR;
  List->mitems= mitems;
  return UDM_OK;
}


static void
UdmURLIPopListAdd(UDM_URLIPOPLIST *List, const UDM_URLIPOP *Item)
{
  List->Item[List->nitems++]= *Item;
}


static int
cmp_urlipop(const UDM_URLIPOP *a, const UDM_URLIPOP *b)
{
  if (a->popularity < b->popularity)
    return 1;
  if (a->popularity > b->popularity)
    return -1;
  if (a->id < b->id)
    return -1;
  if (a->id > b->id)
    return 1;
  return 0;
}

static void
UdmURLIPopListSort(UDM_URLIPOPLIST *List)
{
  UdmSort(List->Item, List->nitems, sizeof(UDM_URLIPOP), (udm_qsort_cmp) cmp_urlipop);
}


static size_t
UdmURLIPopListCountSame(UDM_URLIPOPLIST *List, size_t offs)
{
  size_t offs0= offs++;
  for ( ; offs < List->nitems; offs++)
  {
    if (List->Item[offs0].popularity != List->Item[offs].popularity)
      break;
  }
  return offs - offs0;
}


static udm_rc_t
UdmURLIPopListEncode(UDM_AGENT *A, UDM_URLIPOPLIST *List, UDM_DSTR *dstr)
{
  size_t i;
  for (i= 0; i < List->nitems; )
  {
    size_t n= UdmURLIPopListCountSame(List, i);
    urlid_t id;
    if (!UdmDSTRAppendINT2BE(dstr, List->Item[i].popularity))
      return UDM_ERROR;
    if (UDM_OK != UdmDSTRAppendCoord(dstr, n))
    {
      UdmLog(A, UDM_LOG_ERROR,
             "URLIPopListEncode: DSTRAppendCoord failed: count=%d",
             (int) n);
      return UDM_ERROR;
    }
    for (id= 0; n ; n--)
    {
      UDM_URLIPOP *Item= &List->Item[i++];
      if (UDM_OK != UdmDSTRAppendCoord(dstr, Item->id - id))
      {
        UdmLog(A, UDM_LOG_ERROR,
               "URLIPopListEncode: DSTRAppendCoord failed: delta=%d",
               Item->id - id);
        return UDM_ERROR;
      }
      id= Item->id;
    }
  }
  return UDM_OK;
}


#define ERROR_BYTES_TO_SHOW 4

udm_rc_t
UdmURLDataListUnpackPopularity(UDM_AGENT *A, UDM_URLDATALIST *URLDataList,
                               UDM_CONST_STR *cstr)
{
  const char *str= cstr->str;
  const char *end= cstr->str + cstr->length;
  size_t nfound;
  char hex[ERROR_BYTES_TO_SHOW + 1];
  for (nfound= 0; str + 3 < end ; )
  {
    urlid_t id;
    size_t ndocs, nbytes, i;
    uint4 pop= (((uint4) (unsigned char) str[0]) << 8) + (unsigned char) str[1];
    str+= 2;
    str+= (nbytes= udm_coord_get(&ndocs, (const unsigned char *) str,
                                         (const unsigned char *) end));
    if (!nbytes)
      goto err;
    /*fprintf(stderr, "pop=%d ndocs=%d\n", (int) pop, (int) ndocs);*/
    for (id= 0, i= 0; i < ndocs; i++)
    {
      UDM_URLDATA *data;
      size_t delta;
      str+= (nbytes= udm_coord_get(&delta, (const unsigned char *) str,
                                           (const unsigned char *) end));
      if (!nbytes)
        goto err;
      id+= delta;
      if ((data= UdmURLDataListSearch(URLDataList, id)))
      {
        data->pop_rank= (double) pop / (double) 0xFFFF;
        nfound++;
      }
      /*fprintf(stderr, "  delta=%d id=%d %p\n", (int) delta, (int) id, (void*)data);*/
    }
  }
  UdmLog(A, UDM_LOG_DEBUG, "Found %d documents in '##pop' record", (int) nfound);
  return UDM_OK;
err:
  UdmHexEncode(hex, str, UDM_MIN(end - str, ERROR_BYTES_TO_SHOW));
  UdmLog(A, UDM_LOG_ERROR, "Bad data format in '##pop' record (%s)", hex);
  return UDM_ERROR;
}


static udm_rc_t
UdmURLIPopListInitFromURLDataList(UDM_URLIPOPLIST *IPopList,
                                  UDM_URLDATALIST *List)
{
  size_t i;
  UdmURLIPopListInit(IPopList);
  if (UDM_OK != UdmURLIPopListAlloc(IPopList, List->nitems))
    return UDM_ERROR;
  for (i= 0; i < List->nitems; i++)
  {
    UDM_URLDATA *Item= &List->Item[i];
    UDM_URLIPOP tmp;
    if ((tmp.popularity= (int) (Item->pop_rank * 0xFFFF)))
    {
      if (tmp.popularity > 0xFFFF)
        tmp.popularity= 0xFFFF;
      tmp.id= Item->url_id;
      UdmURLIPopListAdd(IPopList, &tmp);
    }
  }
  return UDM_OK;
}


udm_rc_t
UdmURLDataListPackPopularity(UDM_AGENT *A, UDM_URLDATALIST *List, UDM_DSTR *pop)
{
  UDM_URLIPOPLIST IPopList;

  if (UDM_OK != UdmURLIPopListInitFromURLDataList(&IPopList, List))
    return UDM_ERROR;

  if (IPopList.nitems)
  {
    UdmURLIPopListSort(&IPopList);
    if (UDM_OK != UdmURLIPopListEncode(A, &IPopList, pop))
      return UDM_ERROR;
    /*
    for (i= 0; i < IPopList.nitems; i++)
    {
      UDM_URLIPOP *Item= &IPopList.Item[i];
      fprintf(stderr, "[%d]=%d %04X\n", Item->id, Item->popularity, Item->popularity);
    }
    */
  }
  UdmURLIPopListFree(&IPopList);
  return UDM_OK;
}