ifile-1.3.8/include/ifile.h

/* This file is ifile.h - main header file containing all global variables,
   function prototypes and structure declarations */

/* ifile - intelligent mail filter for EXMH/MH
   Copyright (C) 1997  Jason Daniel Rennie <jr6b+@andrew.cmu.edu>
   Unless otherwise specified, written by Jason Daniel Rennie

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; either version 2
   of the License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
   */

/* NOTE: some portions taken/adapted from libbow - written by Andrew Kachites
 * McCallum */

#ifndef __IFILE_H_
#define __IFILE_H_

#include <stdio.h>
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include "argp/argp.h"
#include "extendable_array.h"
#include "hash_table.h"


#define IFILE_VERSION "ifile 1.3.8"
#define IFILE_MAJOR_VERSION 1
#define IFILE_MINOR_VERSION 3
#define IFILE_TRIFLING_VERSION 8

#define FALSE 0
#define TRUE 1

#ifndef MAX_STR_LEN
#define MAX_STR_LEN 2048
#endif

#ifndef IFILE_INIT_FOLDERS
#define IFILE_INIT_FOLDERS 10
#endif

#ifndef IFILE_INIT_WORDS
#define IFILE_INIT_WORDS 5000
#endif

#ifndef IFILE_MAX_WORD_LENGTH
#define IFILE_MAX_WORD_LENGTH 2048
#endif

#define ifile_malloc(x) malloc(x)
#define ifile_realloc(x,y) realloc(x,y)
#define ifile_fopen(x,y) fopen(x,y)

#ifndef DEFAULT_DB_FILE
#define DEFAULT_DB_FILE  ".idata"
#endif

#ifndef CLOCKS_PER_SECOND
  #ifdef CLOCKS_PER_SEC
  #define CLOCKS_PER_SECOND CLOCKS_PER_SEC
  #else
    #ifdef CLK_TCK
    #define CLOCKS_PER_SECOND CLK_TCK
    #else
    #define CLOCKS_PER_SECOND 100
    #endif
  #endif
#endif

#define ALPHA_LEXER 1      /* word is a string of alphabetic characters */
#define WHITE_LEXER 2      /* word is a whitespace separated string */
#define ALPHA_ONLY_LEXER 3 /* word is whitespace separated alpha string */

#ifndef ERROR
#define ERROR -1    /* standard return value for when something goes wrong */
#endif

/* Progress and error reporting.  Setting in error.c. */
/* Adapted from libbow - written by Andrew Kachites McCallum */
enum ifile_verbosity_levels {
  ifile_silent = 0, /* only fatal errors */
  ifile_quiet,      /* only warnings and errors */
  ifile_progress,   /* enough lines to show progress */
  ifile_verbose,    /* lots of status info */
  ifile_debug       /* everything (and then some) */
};

/* A linked list of digits */
struct linked_list {
  struct linked_list * next;
  int digit;
};

typedef struct linked_list linked_list;

/* struct used when returning categorization of document */
typedef struct _category_rating
{
  double rating;
  char * category;
} category_rating;

/* entry for each word of the database */
typedef struct _db_word_entry {
  char * word;
  long int age;
  long int tot_freq;
  /* int * freq; */
  extendable_array *freq;
} db_word_entry;

/* structure to hold ifile database information */
typedef struct _ifile_db {
  long int num_folders;
  long int num_words;
  long int total_docs;
  long int total_freq;
  long int (*trim_freq)(long int);
  extendable_array folder_name;
  extendable_array folder_freq;
  extendable_array folder_msg;
  htable data;  /* index = (char *)  entry = (db_word_entry *) */
} ifile_db;

/* Used by opts.c to communicate with parse_opt.  */
typedef struct _arguments
{
  extendable_array file;		/* [FILE...] */
  int num_files, thresh;
  int query, query_insert, concise;
  int stemming, stoplist, lexer;
  int skip_header, tag_headers, keep_infrequent, verbosity;
  int max_length;  /* Ignore characters after first MAX_LENGTH characters */
  int print_tokens;  /* Tokenize and print messages - nothing else */
  char *folder_calcs;
  char *minus_folder, *plus_folder;
  char *loocv_folder;
  int create_folder; /* create folder if it does not exist? */
  char *db_file;
  int tmp_file;    /* create a /tmp/ifile.log.<userid> file? */
  int reset_data;
  int occur;
  int read_db, write_db, read_message;  /* boolean - what do we need to do? */
} arguments;

/* initialization functions */
void ifile_db_init (ifile_db * idata);
void ifile_db_entry_init (db_word_entry * wentry);

void ifile_db_free(ifile_db *idata);

/* utility functions */
unsigned long hash(const char * s, long int size);
char * ifile_sprintf (char * format, ...);
char * ifile_cats (long int num_strings, ...);
char * itoa (long int number);
char * readline (char ** bufp);
void ifile_free (void * var);
char * ifile_strdup (const char *s1);
void ifile_bitify_document(htable * message);


/* rating functions */
category_rating * ifile_rate_categories (htable * message, ifile_db * idata);
void ifile_free_categories(category_rating *cr, ifile_db *idata);
void ifile_concise_ratings (char * path, FILE * FP, category_rating * ratings,
			  ifile_db * idata, int thresh);
void ifile_print_ratings (FILE * FP, category_rating * ratings,
			  ifile_db * idata, int thresh);

/* database functions */
void ifile_db_init(ifile_db * idata);
htable * ifile_read_message (FILE * FP);
void ifile_print_message (htable * message);
long int ifile_read_header (ifile_db * idata, char ** bufp);
long int ifile_read_word_frequencies (ifile_db * idata, char ** bufp);
long int ifile_read_word_entry (char * line, ifile_db * idata);
long int ifile_read_db (char * data_file, ifile_db * idata);
long int ifile_write_db (char * data_file, ifile_db * idata);
long int ifile_write_header (FILE * DATA, ifile_db * idata);
long int ifile_write_word_frequencies (FILE * DATA, ifile_db * idata);
long int ifile_age_words (ifile_db * idata, long int epochs);
void ifile_add_db (char * folder, htable * message, ifile_db * idata, int create);
void ifile_del_db (char * folder, htable * message, ifile_db * idata);


/* error handling and logging functions */
char * ifile_strip_path(char * full_path);
FILE * ifile_open_log (int argc, char ** argv);
void ifile_close_log ();
int ifile_verbosify (int verbosity_level, const char *format, ...);
void ifile_error (const char *format, ...);

/* command-line argument functions */
void ifile_init_args (arguments * args);


/*
 * lexing stuff
 */

/* A structure for maintaining the context of a lexer.  (If you need
   to create a lexer that uses more context than this, define a new
   structure that includes this structure as its first element;
   IFILE_LEX_GRAM, defined below is an example of this.)  */
/* Adapted from libbow - written by Andrew Kachites McCallum */
typedef struct _ifile_lex {
  char *document;
  int document_length;
  int document_position;
} ifile_lex;

/* A lexer is represented by a pointer to a structure of this type. */
/* sizeof_lex - size of corresponding _ifile_lex structure
 * *open_text_fp - function to open the document to be lexed
 * *get_word - function for getting the next word in the document
 * *close - function for closing the document
 * document_start_pattern - string to indicate the beginning of the
 *   document (within the file)
 * document_end_pattern - string to indicate the end of the document
 *   (within the file)
 * note: NULL does not scan forward, "" scans forward to EOF
 */
/* Adapted from libbow - written by Andrew Kachites McCallum */
typedef struct _ifile_lexer {
  int sizeof_lex;
  ifile_lex* (*open_text_fp) (struct _ifile_lexer *self, FILE *fp);
  int (*get_word) (struct _ifile_lexer *self, ifile_lex *lex,
		   char *buf, int buflen);
  void (*close) (struct _ifile_lexer *self, ifile_lex *lex);
  const char *document_start_pattern;
  const char *document_end_pattern;
} ifile_lexer;

/* This is an augmented version of IFILE_LEXER that works for simple,
   context-free lexers. */
/* Adapted from libbow - written by Andrew Kachites McCallum */
typedef struct _ifile_lexer_simple {
  /* The basic lexer. */
  ifile_lexer lexer;
  /* Parameters of the simple, context-free lexing. */
  int (*true_to_start)(int character);          /* non-zero on char to start */
  int (*false_to_end)(int character);           /* zero on char to end */
  int (*stoplist_func)(const char *);           /* one on token in stoplist */
  int (*stem_func)(char *);	                /* modify arg by stemming */
  int case_sensitive;		                /* boolean */
  int strip_non_alphas_from_end;                /* boolean */
  int toss_words_containing_non_alphas;	        /* boolean */
  int toss_words_containing_this_many_digits;
  int toss_words_longer_than;
} ifile_lexer_simple;

/* Get the raw token from the document buffer by scanning forward
   until we get a start character, and filling the buffer until we get
   an ending character.  The resulting token in the buffer is
   NULL-terminated.  Return the length of the token. */
int ifile_lexer_simple_get_raw_word (ifile_lexer_simple *self, ifile_lex *lex,
				     char *buf, int buflen);

/* Perform all the necessary postprocessing after the initial token
   boundaries have been found: strip non-alphas from end, toss words
   containing non-alphas, toss words containing certaing many digits,
   toss words appearing in the stop list, stem the word, check the
   stoplist again, toss words of length one.  If the word is tossed,
   return zero, otherwise return the length of the word. */
int ifile_lexer_simple_postprocess_word (ifile_lexer_simple *self,
					 ifile_lex *lex, char *buf, int buflen);

/* Create and return a IFILE_LEX, filling the document buffer from
   characters in FP, starting after the START_PATTERN, and ending with
   the END_PATTERN. */
ifile_lex *ifile_lexer_simple_open_text_fp (ifile_lexer *self, FILE *fp);

/* Close the LEX buffer, freeing the memory held by it. */
void ifile_lexer_simple_close (ifile_lexer *self, ifile_lex *lex);

/* Scan a single token from the LEX buffer, placing it in BUF, and
   returning the length of the token.  BUFLEN is the maximum number of
   characters that will fit in BUF.  If the token won't fit in BUF,
   an error is raised. */
int ifile_lexer_simple_get_word (ifile_lexer *self, ifile_lex *lex,
			       char *buf, int buflen);

/* A lexer that throws out all space-delimited strings that have any
   non-alphabetical characters.  For example, the string `obtained
   from http://www.cs.cmu.edu' will result in the tokens `obtained'
   and `from', but the URL will be skipped. */
extern const ifile_lexer_simple *ifile_alpha_only_lexer;

/* A lexer that keeps all alphabetic strings, delimited by
   non-alphabetic characters.  For example, the string
   `http://www.cs.cmu.edu' will result in the tokens `http', `www',
   `cs', `cmu', `edu'. */
extern const ifile_lexer_simple *ifile_alpha_lexer;

/* A lexer that keeps all strings that begin and end with alphabetic
   characters, delimited by white-space.  For example,
   the string `http://www.cs.cmu.edu' will be a single token. */
extern const ifile_lexer_simple *ifile_white_lexer;


/* Some declarations for a generic indirect lexer.  See lex-indirect.c */
typedef struct _ifile_lexer_indirect {
  ifile_lexer lexer;
  ifile_lexer *underlying_lexer;
} ifile_lexer_indirect;

/* Open the underlying lexer. */
ifile_lex *ifile_lexer_indirect_open_text_fp (ifile_lexer *self, FILE *fp);

/* Close the underlying lexer. */
void ifile_lexer_indirect_close (ifile_lexer *self, ifile_lex *lex);


/* Declarations for an e-mail lexer.  See lex-email.c */

/* An augmented version of IFILE_LEXER that allows for removal of certain
 * e-mail headers */
typedef struct _ifile_lexer_email {
  ifile_lexer_indirect indirect_lexer;
  char **headers_to_keep;
  int gram_size;
} ifile_lexer_email;

/* An augmented version of IFILE_LEX that keeps track of the current
 * document section */
typedef struct _ifile_lex_email {
  ifile_lex lex;
  int gram_size_this_time;
} ifile_lex_email;

/* A lexer which selectively throws out headers of an e-mail message */
/* NOTE: value of NULL throws out all headers, value of -1 keeps them all */
extern const ifile_lexer_email *ifile_email_lexer;

/* Some declarations for a simple N-gram lexer.  See lex-gram.c */

/* An augmented version of IFILE_LEXER that provides N-grams */
typedef struct _ifile_lexer_gram {
  ifile_lexer_indirect indirect_lexer;
  int gram_size;
} ifile_lexer_gram;

/* An augmented version of IFILE_LEX that works for N-grams */
typedef struct _ifile_lex_gram {
  ifile_lex lex;
  int gram_size_this_time;
} ifile_lex_gram;

/* A lexer that returns N-gram tokens using IFILE_ALPHA_ONLY_LEXER.
   It actually returns all 1-grams, 2-grams ... N-grams, where N is
   specified by GRAM_SIZE.  */
extern const ifile_lexer_gram *ifile_gram_lexer;


/* The default lexer that will be used by various library functions
   like IFILE_WV_NEW_FROM_TEXT_FP().  You should set this variable to
   point at whichever lexer you desire.  If you do not set it, it
   will point at ifile_alpha_lexer. */
extern ifile_lexer *ifile_default_lexer;

/* Default instances of the lexers that can be modified by libbow's
   argp cmdline argument processing. */
extern ifile_lexer_simple *ifile_default_lexer_simple;
extern ifile_lexer_email *ifile_default_lexer_email;

/* initialize the default lexers */
void ifile_default_lexer_init();


/* Functions that may be useful in writing a lexer. */

/* Apply the Porter stemming algorithm to modify WORD.  Return 0 on success. */
int ifile_stem_porter (char *word);

/* A function wrapper around POSIX's `isalpha' macro. */
int ifile_isalpha (int character);

/* A function wrapper around POSIX's `isgraph' macro. */
int ifile_isgraph (int character);

/* Return non-zero if WORD is on the stoplist. */
int ifile_stoplist_present (const char *word);

/* Add to the stoplist the white-space delineated words from FILENAME.
   Return the number of words added.  If the file could not be opened,
   return -1. */
int ifile_stoplist_add_from_file (const char *filename);

/* Add WORD to the stop list. */
void ifile_stoplist_add_word (const char *word);


/*
 * other stuff from libbow
 */

int ifile_fp_is_text (FILE *fp);


/* Managing int->string and string->int mappings. */

typedef struct _ifile_int4str {
  const char **str_array;
  int str_array_length;
  int str_array_size;
  int *str_hash;
  int str_hash_size;
} ifile_int4str;

/* Allocate, initialize and return a new int/string mapping structure.
   The parameter CAPACITY is used as a hint about the number of words
   to expect; if you don't know or don't care about a CAPACITY value,
   pass 0, and a default value will be used. */
ifile_int4str *ifile_int4str_new (int capacity);

/* Given a integer INDEX, return its corresponding string. */
const char *ifile_int2str (ifile_int4str *map, int index);

/* Given the char-pointer STRING, return its integer index.  If this is
   the first time we're seeing STRING, add it to the mapping, assign
   it a new index, and return the new index. */
int ifile_str2int (ifile_int4str *map, const char *string);

/* Given the char-pointer STRING, return its integer index.  If STRING
   is not yet in the mapping, return -1. */
int ifile_str2int_no_add (ifile_int4str *map, const char *string);

/* Create a new int-str mapping by lexing words from FILE. */
ifile_int4str *ifile_int4str_new_from_text_file (const char *filename);

/* Write the int-str mapping to file-pointer FP. */
void ifile_int4str_write (ifile_int4str *map, FILE *fp);

/* Return a new int-str mapping, created by reading file-pointer FP. */
ifile_int4str *ifile_int4str_new_from_fp (FILE *fp);

/* Return a new int-str mapping, created by reading FILENAME. */
ifile_int4str *ifile_int4str_new_from_file (const char *filename);

/* Free the memory held by the int-word mapping MAP. */
void ifile_int4str_free (ifile_int4str *map);

/* Free the memory held by the stoplist */
void ifile_stoplist_free();

#ifdef DMALLOC
#include "dmalloc.h"
#endif

#endif