~ubuntu-branches/ubuntu/raring/ifile/raring

/* A simple N-gram lexer. */

/* Copyright (C) 1997 Andrew McCallum

   Written by:  Andrew Kachites McCallum <mccallum@cs.cmu.edu>

   This file is part of the Bag-Of-Words Library, `libbow'.

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Library General Public License
   as published by the Free Software Foundation, version 2.
   
   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Library General Public License for more details.

   You should have received a copy of the GNU Library General Public
   License along with this library; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA */

#ifdef _AIX
#pragma alloca
#endif

#include <ifile.h>

#if HAVE_ALLOCA_H
#include <alloca.h>
#endif

#define SELF ((ifile_lexer_email*)self)
#define LEX ((ifile_lex_email*)lex)

/* String compare function used to compare headers.  Ignores case distinctions
   1 => strings don't match.  0 => strings do match. */
int
strcmp_ignore_case (char *str1, char *str2)
{
  int pos = 0;

  assert(str1 != NULL);  assert(str2 != NULL);

  while (tolower(str1[pos]) == tolower(str2[pos]))
    {
      if (str1[pos] == '\0') { return 0; }
      pos++;
    }

  return 1;
}


/* Given a lex which is pointing to the first character of a header line,
 * determines whether the header should be kept and extracts the portion
 * which should be kept.  Returns NULL if the header should be tossed.
 * Otherwise, returns a pointer to a newly allocated string which is the
 * portion of the header which should be kept.  Modifies
 * lex->document_position so that it points at the first character of the line
 * immediately following the header. */
/* NOTE: function makes some modifications to lex->document */
/* written by Jason Rennie <jrennie@ai.mit.edu> for ifile */
char *
ifile_lexer_email_parse_header (ifile_lex *lex, ifile_lexer *self)
{
  int keep_header;         /* boolean */
  int start_header_name = lex->document_position;
  int start_header;        /* first character after the colon */
  char byte, next_byte;
  char *ret;               /* header string to return */
  int ret_size;
  int i;

  /*
  char buf[80];
  strncpy(buf, &lex->document[lex->document_position], 79);
  buf[79] = '\0';
  printf("ifile_lexer_email_parse_header (lex=%p, self=%p)\n%s\n",
	 lex, self, buf);
	 */

  /* find the colon, indicating the end of the header name */
  do
    {
      byte = lex->document[lex->document_position++];
    }
  while (((byte >= 33 && byte <= 57) || (byte >= 59 && byte <= 126))
	 && lex->document_position < lex->document_length);

  start_header = lex->document_position;
  keep_header = FALSE;
  /* check to see if we should be keeping this header */
  if (byte == 58)
    {
      lex->document[lex->document_position-1] = '\0';
      ifile_verbosify(ifile_debug, "Checking \"%s\"... ",
		      &lex->document[start_header_name]);
      for (i = 0; !keep_header && SELF->headers_to_keep[i]; i++)
	if (strcmp_ignore_case (SELF->headers_to_keep[i],
				&lex->document[start_header_name]) == 0)
	  {
	    keep_header = TRUE;
	    ifile_verbosify(ifile_debug, "okay.\n");
	  }
    }
  else
    {
      /* this doesn't appear to be a header */
      ifile_verbosify(ifile_debug, "not a header.\n");
      return (char *) ERROR;
    }

  /* search forward until the end of the header is reached */
  next_byte = lex->document[lex->document_position];
  do
    {
      byte = next_byte;
      next_byte = lex->document[++lex->document_position];
    }
  while ((byte != '\n'
	  || next_byte == 32
	  || next_byte == 9)
	 && byte != '\0'
	 && lex->document_position < lex->document_length);

  if (keep_header == FALSE)
    {
      ifile_verbosify(ifile_debug, "tossed.\n");
      return NULL;
    }

  /* make a new string out of the portion of the header we want to keep */
  ret_size = lex->document_position - start_header;
  ret = malloc(sizeof(char) * (ret_size+1));
  strncpy(ret, &lex->document[start_header], ret_size);
  ret[ret_size] = '\0';

  return ret;

}

/* Does a pre-lexing of the headers, removing certain headers and eliminating
 * the tags from others */
/* written by Jason Rennie <jrennie@ai.mit.edu> for ifile */
int
ifile_lexer_email_prelex_header (ifile_lex *lex, ifile_lexer *self)
{
  int begin_unfinished;  /* first character of the unfinished section */
  char *header, *body;
  int body_size;

  /*
  printf("ifile_lexer_email_prelex_header(lex=%p, self=%p)\n", lex, self);
  */

  if (SELF->headers_to_keep == NULL
      || SELF->headers_to_keep == (char **) -1)
    return 0;
  if (lex->document_position != 0)
    return ERROR;

  begin_unfinished = 0;

  /* check each header, removing any unwanted portions from the document */
  while (lex->document_position < lex->document_length
	 && lex->document[lex->document_position] != '\n')
    {
      if ((header = ifile_lexer_email_parse_header (lex, self)))
	{
	  if (header == (char *) ERROR) break;
	  strcpy (&lex->document[begin_unfinished], header);
	  begin_unfinished += strlen (header);
	  free (header);
	}
    }

  body_size = lex->document_length - lex->document_position;
  body = (char *) malloc (sizeof(char) * (body_size + 1));
  strncpy (body, &lex->document[lex->document_position], body_size);
  body[body_size] = '\0';
  strcpy (&lex->document[begin_unfinished], body);
  lex->document_length = begin_unfinished + body_size - 1;

  lex->document_position = 0;
  return 0;
}

ifile_lex *
ifile_lexer_email_open_text_fp (ifile_lexer *self, FILE *fp)
{
  ifile_lex *lex;

  /* to toss all headers, don't begin document until the first blank line */
  if (SELF->headers_to_keep == NULL)
    self->document_start_pattern = "\n\n";
  lex = ifile_lexer_indirect_open_text_fp (self, fp);
  if (lex == NULL)
    return NULL;
  LEX->gram_size_this_time = SELF->gram_size;
  if (ifile_lexer_email_prelex_header (lex, self))
    ifile_verbosify(ifile_progress, "Did not pre-lex e-mail headers properly!\n");
  
  return lex;
}

int
ifile_lexer_email_get_word (ifile_lexer *self, ifile_lex *lex, 
			 char *buf, int buflen)
{
  int i;
  char **tokens;
  int s;
  int len;
  
  tokens = alloca (sizeof (char*) * LEX->gram_size_this_time);
  for (i = 0; i < LEX->gram_size_this_time; i++)
    tokens[i] = alloca (IFILE_MAX_WORD_LENGTH);

  /* Remember where we started. */
  s = LEX->lex.document_position;

  /* Get the first token. */
  if (SELF->indirect_lexer.underlying_lexer->get_word 
      (SELF->indirect_lexer.underlying_lexer, lex,
       tokens[0], IFILE_MAX_WORD_LENGTH)
      == 0)
    return 0;

  /* Get the next n-1 tokens. */
  for (i = 1; i < LEX->gram_size_this_time; i++)
    if (SELF->indirect_lexer.underlying_lexer->get_word
	(SELF->indirect_lexer.underlying_lexer, lex,
	 tokens[i], IFILE_MAX_WORD_LENGTH)
	== 0)
      *(tokens[i]) = '\0';

  /* Make sure it will fit. */
  for (i = 0, len = 0; i < LEX->gram_size_this_time; i++)
    len += strlen (tokens[i]) + 1;
  assert (len < IFILE_MAX_WORD_LENGTH);

  /* Fill buf with the tokens concatenated. */
  strcpy (buf, tokens[0]);
  for (i = 1; i < LEX->gram_size_this_time; i++)
    {
      strcat (buf, ";");
      strcat (buf, tokens[i]);
    }

  /* Put us back to the second token so we can get it with the next call */
  if (LEX->gram_size_this_time > 1)
    LEX->lex.document_position = s;

  if (LEX->gram_size_this_time == 1)
    LEX->gram_size_this_time = SELF->gram_size;
  else
    LEX->gram_size_this_time--;

  return strlen (buf);
}

/* This is declared in lex-simple.c */
extern ifile_lexer_simple _ifile_alpha_lexer;

const ifile_lexer_email _ifile_email_lexer =
{
  {
    {
      sizeof (ifile_lex_email),
      ifile_lexer_email_open_text_fp,
      ifile_lexer_email_get_word,
      ifile_lexer_indirect_close,
      "",			/* document start pattern begins right away */
      NULL			/* document end pattern goes to end */
    },
    (ifile_lexer*)&_ifile_alpha_lexer, /* default UNDERLYING_LEXER */
  },
  (char **) -1,                 /* keep all headers by default */
  1				/* default gram-size is 1 */
};
const ifile_lexer_email *ifile_email_lexer = &_ifile_email_lexer;