/*   cspeedtest.c
* ===========================================================================
*
*                            PUBLIC DOMAIN NOTICE
*            National Center for Biotechnology Information (NCBI)
*
*  This software/database is a "United States Government Work" under the
*  terms of the United States Copyright Act.  It was written as part of
*  the author's official duties as a United States Government employee and
*  thus cannot be copyrighted.  This software/database is freely available
*  to the public for use. The National Library of Medicine and the U.S.
*  Government do not place any restriction on its use or reproduction.
*  We would, however, appreciate having the NCBI and the author cited in
*  any work or product based on this material
*
*  Although all reasonable efforts have been taken to ensure the accuracy
*  and reliability of the software and data, the NLM and the U.S.
*  Government do not and cannot warrant the performance or results that
*  may be obtained by using this software or data. The NLM and the U.S.
*  Government disclaim all warranties, express or implied, including
*  warranties of performance, merchantability or fitness for any particular
*  purpose.
*
* ===========================================================================
*
* File Name:  cspeedtest.c
*
* Author:  Jonathan Kans
*
* Version Creation Date:   12/17/07
*
* $Revision: 1.25 $
*
* File Description: 
*
* Modifications:  
* --------------------------------------------------------------------------
* Date     Name        Description of modification
* -------  ----------  -----------------------------------------------------
*
*
* ==========================================================================
*/

#include <ncbi.h>
#include <objall.h>
#include <objsset.h>
#include <objfdef.h>
#include <objsub.h>
#include <sequtil.h>
#include <sqnutils.h>
#include <explore.h>
#include <toasn3.h>
#include <pmfapi.h>
#include <tofasta.h>
#include <asn2gnbk.h>
#include <valid.h>
#include <suggslp.h>

#define CSPEEDTEST_APP_VER "2.1"

CharPtr CSPEEDTEST_APPLICATION = CSPEEDTEST_APP_VER;

typedef struct cspeedflags {
  Boolean       batch;
  Boolean       binary;
  Boolean       compressed;
  Boolean       lock;
  Int2          type;
  Int4          maxcount;
  CharPtr       io;
  CharPtr       clean;
  CharPtr       skip;
  CharPtr       index;
  CharPtr       seq;
  CharPtr       feat;
  CharPtr       desc;
  CharPtr       verify;
  BioseqPtr     nucbsp;
  Int2          genCode;
  AsnModulePtr  amp;
  AsnTypePtr    atp_bss;
  AsnTypePtr    atp_bsss;
  AsnTypePtr    atp_se;
  AsnTypePtr    atp_bsc;
  AsnTypePtr    bssp_atp;
  BioseqSet     bss;
  FILE          *ofp;
  FILE          *logfp;
} CSpeedFlagData, PNTR CSpeedFlagPtr;

static void DoVisitFeaturesTest (
  SeqFeatPtr sfp,
  Pointer userdata
)

{
  /* empty visit callback */
}

static void DoVisitCodingRegions (
  SeqFeatPtr sfp,
  Pointer userdata
)

{
  BioseqPtr      bsp;
  CharPtr        caret5, caret3;
  CSpeedFlagPtr  cfp;
  Char           id [64];
  SeqLocPtr      loc, slp;
  Boolean        partial5, partial3;
  SeqIdPtr       sip;
  Int4           start, stop;

  if (sfp == NULL || sfp->data.choice != SEQFEAT_CDREGION) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL || cfp->ofp == NULL) return;

  loc = sfp->location;
  bsp = BioseqFindFromSeqLoc (loc);
  if (bsp == NULL) return;

  StringCpy (id, "?");
  if (sfp->product != NULL) {
    sip = SeqLocId (sfp->product);
    if (sip != NULL) {
      SeqIdWrite (sip, id, PRINTID_FASTA_SHORT, sizeof (id) - 1);
    }
  }

  fprintf (cfp->ofp, "%s\n", id);
  slp = SeqLocFindNext (loc, NULL);
  while (slp != NULL) {
    start = GetOffsetInBioseq (slp, bsp, SEQLOC_START) + 1;
    stop = GetOffsetInBioseq (slp, bsp, SEQLOC_STOP) + 1;
    caret5 = "";
    caret3 = "";
    CheckSeqLocForPartial (slp, &partial5, &partial3);
    if (partial5) {
      caret5 = "<";
    }
    if (partial3) {
      caret3 = ">";
    }
    fprintf (cfp->ofp, "%s%ld\t%s%ld\n", caret5, (long) start, caret3, (long) stop);
    slp = SeqLocFindNext (loc, slp);
  }
}

static void DoSuggestIntervals (
  BioseqPtr bsp,
  Pointer userdata
)

{
  CharPtr        caret5, caret3;
  CSpeedFlagPtr  cfp;
  Char           id [64];
  SeqLocPtr      loc, slp;
  Boolean        partial5, partial3;
  SeqAnnotPtr    sap;
  SeqFeatPtr     sfp;
  SeqIdPtr       sip;
  Int4           start, stop;

  if (bsp == NULL) return;
  if (! ISA_aa (bsp->mol)) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL || cfp->ofp == NULL || cfp->nucbsp == NULL) return;

  sip = SeqIdFindBest (bsp->id, 0);
  if (sip == NULL) return;
  SeqIdWrite (sip, id, PRINTID_FASTA_SHORT, sizeof (id) - 1);

  sap = SuggestCodingRegion (cfp->nucbsp, bsp, cfp->genCode);
  if (sap == NULL) return;
  if (sap->type == 1) {
    sfp = (SeqFeatPtr) sap->data;
    if (sfp != NULL && sfp->data.choice == SEQFEAT_CDREGION) {
      loc = sfp->location;
      if (loc != NULL) {
        fprintf (cfp->ofp, "%s\n", id);
        slp = SeqLocFindNext (loc, NULL);
        while (slp != NULL) {
          start = GetOffsetInBioseq (slp, cfp->nucbsp, SEQLOC_START) + 1;
          stop = GetOffsetInBioseq (slp, cfp->nucbsp, SEQLOC_STOP) + 1;
          caret5 = "";
          caret3 = "";
          CheckSeqLocForPartial (slp, &partial5, &partial3);
          if (partial5) {
            caret5 = "<";
          }
          if (partial3) {
            caret3 = ">";
          }
          fprintf (cfp->ofp, "%s%ld\t%s%ld\n", caret5, (long) start, caret3, (long) stop);
          slp = SeqLocFindNext (loc, slp);
        }
      }
    }
  }
  SeqAnnotFree (sap);
}

static void DoGeneOverlapPrintTest (
  SeqFeatPtr sfp,
  Pointer userdata
)

{
  CSpeedFlagPtr      cfp;
  SeqMgrFeatContext  gcontext;
  SeqFeatPtr         gene;
  CharPtr            str1, str2;

  if (sfp == NULL) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL || cfp->ofp == NULL) return;

  if (sfp->data.choice == SEQFEAT_GENE) return;
  gene = SeqMgrGetOverlappingGene (sfp->location, &gcontext);
  if (gene == NULL) return;

  str1 = SeqLocPrint (sfp->location);
  str2 = SeqLocPrint (gene->location);
  if (str1 != NULL && str2 != NULL) {
    fprintf (cfp->ofp, "[%s] -> [%s]\n", str1, str2);
  } else {
    fprintf (cfp->ofp, "? -> ?\n");
  }
  MemFree (str1);
  MemFree (str2);
}

static void DoGeneOverlapSpeedTest (
  SeqFeatPtr sfp,
  Pointer userdata
)

{
  CSpeedFlagPtr      cfp;
  SeqMgrFeatContext  gcontext;
  SeqFeatPtr         gene;

  if (sfp == NULL) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL || cfp->ofp == NULL) return;

  if (sfp->data.choice == SEQFEAT_GENE) return;
  gene = SeqMgrGetOverlappingGene (sfp->location, &gcontext);
  if (gene == NULL) return;
}

static void LIBCALLBACK EmptyStreamProc (
  CharPtr sequence,
  Pointer userdata
)

{
  /* empty stream callback */
}

static void DoFastaSeq (
  BioseqPtr bsp,
  Pointer userdata
)

{
  CSpeedFlagPtr  cfp;

  if (bsp == NULL) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL) return;

  if (cfp->ofp != NULL) {
    BioseqFastaStream (bsp, cfp->ofp, STREAM_EXPAND_GAPS, 70, 0, 0, TRUE);
  } else {
    SeqPortStream (bsp, STREAM_EXPAND_GAPS, NULL, EmptyStreamProc);
  }
}

static void DoFastaRaw (
  BioseqPtr bsp,
  Pointer userdata
)

{
  CSpeedFlagPtr  cfp;

  if (bsp == NULL) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL) return;

  if (cfp->ofp != NULL) {
    fprintf (cfp->ofp, ">\n");
    BioseqFastaStream (bsp, cfp->ofp, STREAM_EXPAND_GAPS, 70, 0, 0, FALSE);
  } else {
    SeqPortStream (bsp, STREAM_EXPAND_GAPS, NULL, EmptyStreamProc);
  }
}

static void DoFastaDefline (
  BioseqPtr bsp,
  Pointer userdata
)

{
  Char           buf [4096];
  CSpeedFlagPtr  cfp;
  Char           id [128];

  if (bsp == NULL) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL) return;

  id [0] = '\0';
  SeqIdWrite (bsp->id, id, PRINTID_FASTA_LONG, sizeof (id) - 1);
  buf [0] = '\0';
  CreateDefLine (NULL, bsp, buf, sizeof (buf) - 1, 0, NULL, NULL);

  if (cfp->ofp != NULL) {
    fprintf (cfp->ofp, ">%s %s\n", id, buf);
  }
}

static void DoNewFastaDefline (
  BioseqPtr bsp,
  Pointer userdata,
  Boolean ignoreExisting
)

{
  BioseqSetPtr   bssp;
  CSpeedFlagPtr  cfp;
  Char           id [128];
  CharPtr        title;

  if (bsp == NULL) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL) return;

  if (StringChr (cfp->skip, 's') != NULL) {
    if (bsp->idx.parenttype == OBJ_BIOSEQSET) {
      bssp = (BioseqSetPtr) bsp->idx.parentptr;
      if (bssp != NULL) {
        if (bssp->_class == BioseqseqSet_class_segset ||
            bssp->_class == BioseqseqSet_class_parts) return;
      }
    }
  }
  if (StringChr (cfp->skip, 'v') != NULL) {
    if (bsp->repr == Seq_repr_virtual) return;
  }

  id [0] = '\0';
  SeqIdWrite (bsp->id, id, PRINTID_FASTA_LONG, sizeof (id) - 1);
  title = NewCreateDefLine (NULL, bsp, ignoreExisting, FALSE);
  if (StringHasNoText (title)) {
    title = StringSave ("?");
  }

  if (cfp->ofp != NULL) {
    fprintf (cfp->ofp, ">%s %s\n", id, title);
  }

  MemFree (title);
}

static void DoNewFastaExist (
  BioseqPtr bsp,
  Pointer userdata
)

{
  DoNewFastaDefline (bsp, userdata, FALSE);
}

static void DoNewFastaRegen (
  BioseqPtr bsp,
  Pointer userdata
)

{
  DoNewFastaDefline (bsp, userdata, TRUE);
}

static void DoFastaComp (
  BioseqPtr bsp,
  Pointer userdata,
  Boolean ignoreExisting
)

{
  Char           buf [4096];
  CSpeedFlagPtr  cfp;
  Char           id [128];
  CharPtr        title;

  if (bsp == NULL) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL) return;

  id [0] = '\0';
  SeqIdWrite (bsp->id, id, PRINTID_FASTA_LONG, sizeof (id) - 1);
  buf [0] = '\0';
  CreateDefLineExEx (NULL, bsp, buf, sizeof (buf) - 1, 0,
                     NULL, NULL, ignoreExisting, FALSE);
  title = NewCreateDefLine (NULL, bsp, ignoreExisting, FALSE);
  if (StringHasNoText (title)) {
    title = StringSave ("?");
  }

  if (StringCmp (buf, title) != 0) {
    if (cfp->ofp != NULL) {
      fprintf (cfp->ofp, "<  %s %s\n", id, buf);
      fprintf (cfp->ofp, ">  %s %s\n", id, title);
    }
    printf ("<  %s %s\n", id, buf);
    printf (">  %s %s\n", id, title);
    fflush (stdout);
  }

  MemFree (title);
}

static void DoFastaExist (
  BioseqPtr bsp,
  Pointer userdata
)

{
  DoFastaComp (bsp, userdata, FALSE);
}

static void DoFastaRegen (
  BioseqPtr bsp,
  Pointer userdata
)

{
  DoFastaComp (bsp, userdata, TRUE);
}

static void DoFastaFeat (
  SeqFeatPtr sfp,
  Pointer userdata
)

{
  CSpeedFlagPtr  cfp;
  CharPtr        str;

  if (sfp == NULL) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL) return;

  if (cfp->ofp != NULL) {
    str = SeqLocPrint (sfp->location);
    if (str != NULL) {
      fprintf (cfp->ofp, "> [%s]\n", str);
      MemFree (str);
    }
    SeqLocFastaStream (sfp->location, cfp->ofp, STREAM_EXPAND_GAPS, 70, 0, 0);
  } else {
    SeqPortStreamLoc (sfp->location, STREAM_EXPAND_GAPS, NULL, EmptyStreamProc);
  }
}

static void DoFastaTrans (
  SeqFeatPtr sfp,
  Pointer userdata
)

{
  ByteStorePtr   bs;
  CSpeedFlagPtr  cfp;
  CharPtr        seq, str;

  if (sfp == NULL) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL) return;

  if (sfp->data.choice != SEQFEAT_CDREGION) return;
  bs = ProteinFromCdRegion (sfp, FALSE);
  if (bs == NULL) return;

  seq = (CharPtr) BSMerge (bs, NULL);
  BSFree (bs);
  if (seq == NULL) return;

  if (cfp->ofp != NULL) {
    str = SeqLocPrint (sfp->location);
    if (str != NULL) {
      fprintf (cfp->ofp, "> (%s)\n", str);
      MemFree (str);
    }
    fprintf (cfp->ofp, "%s\n", seq);
  }

  MemFree (seq);
}

static CharPtr compatSeverityLabel [] = {
  "NONE", "NOTE: valid", "WARNING: valid", "ERROR: valid", "REJECT: valid", "FATAL: valid", "MAX", NULL
};

static void LIBCALLBACK ValidCallback (
  ErrSev severity,
  int errcode,
  int subcode,
  Uint2 entityID,
  Uint2 itemtype,
  Uint4 itemID,
  CharPtr accession,
  CharPtr featureID,
  CharPtr message,
  CharPtr objtype,
  CharPtr label,
  CharPtr context,
  CharPtr location,
  CharPtr product,
  Pointer userdata
)

{
  CharPtr  catname, errname;
  FILE     *fp;

  fp = (FILE *) userdata;
  if (fp == NULL) return;

  if (severity < SEV_NONE || severity > SEV_MAX) {
    severity = SEV_MAX;
  }

  catname = GetValidCategoryName (errcode);
  errname = GetValidErrorName (errcode, subcode);

  if (catname == NULL) {
    catname = "?";
  }
  if (errname == NULL) {
    errname = "?";
  }

  if (accession == NULL) {
    accession = "";
  }
  if (message == NULL) {
    message = "";
  }
  if (objtype == NULL) {
    objtype = "";
  }
  if (label == NULL) {
    label = "";
  }

  fprintf (fp, "%s [%s.%s] %s %s: %s",
           compatSeverityLabel [severity],
           catname, errname, message, objtype, label);
  if (location != NULL) {
    fprintf (fp, " %s", location);
  }
  if (context != NULL) {
    fprintf (fp, " %s", context);
  }
  if (product != NULL) {
    fprintf (fp, " -> %s", product);
  }
  fprintf (fp, "\n");
}

static void MarkTitles (
  SeqDescrPtr sdp,
  Pointer userdata
)

{
  ObjValNodePtr  ovn;

  if (sdp == NULL || sdp->choice != Seq_descr_title) return;
  if (sdp->extended == 0) return;
  ovn = (ObjValNodePtr) sdp;
  ovn->idx.deleteme = TRUE;
}

static void RemoveTitles (
  SeqEntryPtr sep,
  Pointer mydata,
  Int4 index,
  Int2 indent
)

{
  BioseqPtr         bsp;
  BioseqSetPtr      bssp;
  SeqDescrPtr PNTR  prev = NULL;
  SeqDescrPtr       sdp = NULL, next = NULL;

  if (sep == NULL || sep->data.ptrvalue == NULL) return;
  if (IS_Bioseq (sep)) {
    bsp = (BioseqPtr) sep->data.ptrvalue;
    sdp = bsp->descr;
    prev = &(bsp->descr);
  } else if (IS_Bioseq_set (sep)) {
    bssp = (BioseqSetPtr) sep->data.ptrvalue;
    sdp = bssp->descr;
    prev = &(bssp->descr);
  } else return;

  while (sdp != NULL) {
    next = sdp->next;
    if (sdp->choice == Seq_descr_title) {
      *prev = sdp->next;
      sdp->next = NULL;
      SeqDescFree (sdp);
    } else {
      prev = (SeqDescrPtr PNTR) &(sdp->next);
    }
    sdp = next;
  }
}

static void DoProcess (
  SeqEntryPtr sep,
  Uint2 entityID,
  CSpeedFlagPtr cfp
)

{
  Char            id [64];
  ErrSev          oldErrSev;
  ValidStructPtr  vsp;

  if (sep == NULL || cfp == NULL) return;

  if (StringChr (cfp->clean, 'm') != NULL) {
    VisitDescriptorsInSep (sep, NULL, MarkTitles);
  }
  if (StringChr (cfp->clean, 'd') != NULL) {
    DeleteMarkedObjects (entityID, 0, NULL);
  }
  if (StringChr (cfp->clean, 'c') != NULL) {
    SeqMgrClearFeatureIndexes (entityID, NULL);
  }
  if (StringChr (cfp->clean, 't') != NULL) {
    SeqEntryExplore (sep, NULL, RemoveTitles);
  }
  if (StringChr (cfp->clean, 'a') != NULL) {
    AssignIDsInEntity (entityID, 0, NULL);
  }
  if (StringChr (cfp->clean, 'b') != NULL) {
    BasicSeqEntryCleanup (sep);
  }
  if (StringChr (cfp->clean, 's') != NULL) {
    SeriousSeqEntryCleanup (sep, NULL, NULL);
  }

  if (StringChr (cfp->index, 'f') != NULL) {
    SeqMgrIndexFeatures (entityID, NULL);
  }

  if (StringChr (cfp->seq, 'c') != NULL) {
    VisitBioseqsInSep (sep, (Pointer) cfp, DoFastaExist);
  }
  if (StringChr (cfp->seq, 'C') != NULL) {
    VisitBioseqsInSep (sep, (Pointer) cfp, DoFastaRegen);
  }
  if (StringChr (cfp->seq, 's') != NULL) {
    VisitBioseqsInSep (sep, (Pointer) cfp, DoFastaSeq);
  }
  if (StringChr (cfp->seq, 'S') != NULL) {
    if (SeqMgrFeaturesAreIndexed (entityID) == 0) {
      SeqMgrIndexFeatures (entityID, NULL);
    }
    VisitBioseqsInSep (sep, (Pointer) cfp, DoFastaSeq);
  }
  if (StringChr (cfp->seq, 'r') != NULL) {
    VisitBioseqsInSep (sep, (Pointer) cfp, DoFastaRaw);
  }
  if (StringChr (cfp->seq, 'd') != NULL) {
    VisitBioseqsInSep (sep, (Pointer) cfp, DoFastaDefline);
  }
  if (StringChr (cfp->seq, 'D') != NULL) {
    if (SeqMgrFeaturesAreIndexed (entityID) == 0) {
      SeqMgrIndexFeatures (entityID, NULL);
    }
    VisitBioseqsInSep (sep, (Pointer) cfp, DoFastaDefline);
  }
  if (StringChr (cfp->seq, 'T') != NULL) {
    VisitDescriptorsInSep (sep, NULL, MarkTitles);
    DeleteMarkedObjects (entityID, 0, NULL);
    SeqMgrIndexFeatures (entityID, NULL);
    VisitBioseqsInSep (sep, (Pointer) cfp, DoFastaDefline);
  }
  if (StringChr (cfp->seq, 'x') != NULL) {
    VisitBioseqsInSep (sep, (Pointer) cfp, DoNewFastaExist);
  }
  if (StringChr (cfp->seq, 'X') != NULL) {
    /*
    VisitDescriptorsInSep (sep, NULL, MarkTitles);
    DeleteMarkedObjects (entityID, 0, NULL);
    SeqMgrIndexFeatures (entityID, NULL);
    */
    VisitBioseqsInSep (sep, (Pointer) cfp, DoNewFastaRegen);
  }
  
  if (StringChr (cfp->seq, 'f') != NULL) {
    VisitFeaturesInSep (sep, (Pointer) cfp, DoFastaFeat);
  }
  if (StringChr (cfp->seq, 't') != NULL) {
    VisitFeaturesInSep (sep, (Pointer) cfp, DoFastaTrans);
  }

  if (StringChr (cfp->feat, 'v') != NULL) {
    VisitFeaturesInSep (sep, NULL, DoVisitFeaturesTest);
  }
  if (StringChr (cfp->feat, 'g') != NULL) {
    if (SeqMgrFeaturesAreIndexed (entityID) == 0) {
      SeqMgrIndexFeatures (entityID, NULL);
    }
    VisitFeaturesInSep (sep, (Pointer) cfp, DoGeneOverlapPrintTest);
  }
  if (StringChr (cfp->feat, 'h') != NULL) {
    if (SeqMgrFeaturesAreIndexed (entityID) == 0) {
      SeqMgrIndexFeatures (entityID, NULL);
    }
    VisitFeaturesInSep (sep, (Pointer) cfp, DoGeneOverlapSpeedTest);
  }
  if (StringChr (cfp->feat, 'x') != NULL) {
  }
  if (StringChr (cfp->feat, 'o') != NULL) {
  }
  if (StringChr (cfp->feat, 'd') != NULL) {
  }
  if (StringChr (cfp->feat, 't') != NULL) {
    SeqEntryToGnbk (sep, NULL, FTABLE_FMT, SEQUIN_MODE, NORMAL_STYLE,
                    0, 0, SHOW_PROT_FTABLE, NULL, cfp->ofp);
  }
  if (StringChr (cfp->feat, 's') != NULL) {
    if (SeqMgrFeaturesAreIndexed (entityID) == 0) {
      SeqMgrIndexFeatures (entityID, NULL);
    }
    cfp->nucbsp = FindNucBioseq (sep);
    if (cfp->nucbsp != NULL) {
      BioseqToGeneticCode (cfp->nucbsp, &(cfp->genCode), NULL, NULL, NULL, 0, NULL);
      SeqIdWrite (cfp->nucbsp->id, id, PRINTID_FASTA_LONG, sizeof (id) - 1);
      if (cfp->ofp != NULL) {
        fprintf (cfp->ofp, "%s\n", id);
      }
      VisitBioseqsInSep (sep, (Pointer) cfp, DoSuggestIntervals);
      cfp->nucbsp = NULL;
      cfp->genCode = 0;
    }
  }
  if (StringChr (cfp->feat, 'S') != NULL) {
    if (SeqMgrFeaturesAreIndexed (entityID) == 0) {
      SeqMgrIndexFeatures (entityID, NULL);
    }
    cfp->nucbsp = FindNucBioseq (sep);
    if (cfp->nucbsp != NULL) {
      BioseqToGeneticCode (cfp->nucbsp, &(cfp->genCode), NULL, NULL, NULL, 0, NULL);
      SetBatchSuggestNucleotide (cfp->nucbsp, cfp->genCode);
      SeqIdWrite (cfp->nucbsp->id, id, PRINTID_FASTA_LONG, sizeof (id) - 1);
      if (cfp->ofp != NULL) {
        fprintf (cfp->ofp, "%s\n", id);
      }
      VisitBioseqsInSep (sep, (Pointer) cfp, DoSuggestIntervals);
      ClearBatchSuggestNucleotide ();
      cfp->nucbsp = NULL;
      cfp->genCode = 0;
    }
  }
  if (StringChr (cfp->feat, 'c') != NULL) {
    VisitFeaturesInSep (sep, (Pointer) cfp, DoVisitCodingRegions);
  }

  if (StringChr (cfp->desc, 'b') != NULL) {
  }
  if (StringChr (cfp->desc, 't') != NULL) {
  }

  if (StringChr (cfp->verify, 'v') != NULL) {
    if (SeqMgrFeaturesAreIndexed (entityID) == 0) {
      SeqMgrIndexFeatures (entityID, NULL);
    }
    vsp = ValidStructNew ();
    if (vsp != NULL) {
      vsp->useSeqMgrIndexes = TRUE;
      vsp->suppressContext = TRUE;
      vsp->seqSubmitParent = TRUE;
      vsp->testLatLonSubregion = TRUE;
      oldErrSev = ErrSetMessageLevel (SEV_NONE);
      vsp->errfunc = ValidCallback;
      vsp->userdata = (Pointer) cfp->ofp;
      /* vsp->convertGiToAccn = FALSE; */
      ValidateSeqEntry (sep, vsp);
      ValidStructFree (vsp);
      ErrSetMessageLevel (oldErrSev);
    }
  }
  if (StringChr (cfp->verify, 'b') != NULL) {
    if (SeqMgrFeaturesAreIndexed (entityID) == 0) {
      SeqMgrIndexFeatures (entityID, NULL);
    }
    SeqEntryToGnbk (sep, NULL, GENBANK_FMT, SEQUIN_MODE, NORMAL_STYLE,
                    0, 0, 0, NULL, cfp->ofp);
  }

  if (cfp->ofp != NULL) {
    fflush (cfp->ofp);
  }
}

static void ProcessSingleRecord (
  CharPtr filename,
  CSpeedFlagPtr cfp
)

{
  AsnIoPtr      aip;
  BioseqPtr     bsp;
  ValNodePtr    bsplist = NULL;
  BioseqSetPtr  bssp;
  Pointer       dataptr = NULL;
  Uint2         datatype, entityID = 0;
  FileCache     fc;
  FILE          *fp;
  Int1          iotype;
  Char          line [512];
  Int4          maxio = 1;
  SeqEntryPtr   sep;
  time_t        starttime, stoptime, worsttime;
  CharPtr       str;
  Int4          x;

  if (cfp == NULL) return;

  if (StringHasNoText (filename)) return;

  if (StringChr (cfp->io, 'r') != NULL) {
    maxio = cfp->maxcount;
  }

  starttime = GetSecs ();

  for (x = 0; x < maxio; x++) {
    if (entityID != 0) {
      ObjMgrFreeByEntityID (entityID);
      entityID = 0;
      dataptr = NULL;
    }

    if (cfp->type == 1) {

      fp = FileOpen (filename, "r");
      if (fp == NULL) {
        Message (MSG_POSTERR, "Failed to open '%s'", filename);
        return;
      }

      dataptr = ReadAsnFastaOrFlatFile (fp, &datatype, NULL, FALSE, FALSE, FALSE, FALSE);

      FileClose (fp);

      entityID = ObjMgrRegister (datatype, dataptr);

    } else if (cfp->type >= 2 && cfp->type <= 5) {

      aip = AsnIoOpen (filename, cfp->binary? "rb" : "r");
      if (aip == NULL) {
        Message (MSG_POSTERR, "AsnIoOpen failed for input file '%s'", filename);
        return;
      }

      SeqMgrHoldIndexing (TRUE);
      switch (cfp->type) {
        case 2 :
          dataptr = (Pointer) SeqEntryAsnRead (aip, NULL);
          datatype = OBJ_SEQENTRY;
          break;
        case 3 :
          dataptr = (Pointer) BioseqAsnRead (aip, NULL);
          datatype = OBJ_BIOSEQ;
          break;
        case 4 :
          dataptr = (Pointer) BioseqSetAsnRead (aip, NULL);
          datatype = OBJ_BIOSEQSET;
          break;
        case 5 :
          dataptr = (Pointer) SeqSubmitAsnRead (aip, NULL);
          datatype = OBJ_SEQSUB;
          break;
        default :
          break;
      }
      SeqMgrHoldIndexing (FALSE);

      AsnIoClose (aip);

      entityID = ObjMgrRegister (datatype, dataptr);

    } else if (cfp->type == 6) {

      fp = FileOpen (filename, "r");
      if (fp == NULL) {
        Message (MSG_POSTERR, "Failed to open '%s'", filename);
        return;
      }

      dataptr = ReadAsnFastaOrFlatFile (fp, &datatype, NULL, FALSE, FALSE, FALSE, FALSE);

      FileClose (fp);

      entityID = ObjMgrRegister (datatype, dataptr);

    } else if (cfp->type == 7) {

      fp = FileOpen (filename, "r");
      if (fp == NULL) {
        Message (MSG_POSTERR, "Failed to open '%s'", filename);
        return;
      }

      FileCacheSetup (&fc, fp);

      str = FileCacheReadLine (&fc, line, sizeof (line), NULL);
      while (str != NULL) {
        str = FileCacheReadLine (&fc, line, sizeof (line), NULL);
      }

      FileClose (fp);

      return;

    } else {
      Message (MSG_POSTERR, "Input format type '%d' unrecognized", (int) cfp->type);
      return;
    }
  }

  if (entityID < 1 || dataptr == NULL) {
    Message (MSG_POSTERR, "Data read failed for input file '%s'", filename);
    return;
  }

  if (datatype == OBJ_SEQSUB || datatype == OBJ_SEQENTRY ||
        datatype == OBJ_BIOSEQ || datatype == OBJ_BIOSEQSET) {

    stoptime = GetSecs ();
    worsttime = stoptime - starttime;
    if (cfp->logfp != NULL) {
      fprintf (cfp->logfp, "ASN reading time %ld seconds\n", (long) worsttime);
      fflush (cfp->logfp);
    }

    sep = GetTopSeqEntryForEntityID (entityID);

    if (sep == NULL) {
      sep = SeqEntryNew ();
      if (sep != NULL) {
        if (datatype == OBJ_BIOSEQ) {
          bsp = (BioseqPtr) dataptr;
          sep->choice = 1;
          sep->data.ptrvalue = bsp;
          SeqMgrSeqEntry (SM_BIOSEQ, (Pointer) bsp, sep);
        } else if (datatype == OBJ_BIOSEQSET) {
          bssp = (BioseqSetPtr) dataptr;
          sep->choice = 2;
          sep->data.ptrvalue = bssp;
          SeqMgrSeqEntry (SM_BIOSEQSET, (Pointer) bssp, sep);
        } else {
          sep = SeqEntryFree (sep);
        }
      }
      sep = GetTopSeqEntryForEntityID (entityID);
    }

    if (sep != NULL) {

      if (cfp->lock) {
        starttime = GetSecs ();

        bsplist = LockFarComponents (sep);

        stoptime = GetSecs ();
        worsttime = stoptime - starttime;
        if (cfp->logfp != NULL) {
          fprintf (cfp->logfp, "Far component locking time %ld seconds\n", (long) worsttime);
          fflush (cfp->logfp);
        }
      }

      if (StringChr (cfp->io, 'w') != NULL) {
        starttime = GetSecs ();

        iotype = ASNIO_TEXT_OUT;
        if (StringChr (cfp->io, 'b') != NULL) {
          iotype = ASNIO_BIN_OUT;
        }

        for (x = 0; x < cfp->maxcount; x++) {
          aip = AsnIoNew (iotype, cfp->ofp, NULL, NULL, NULL);
          if (aip != NULL) {
            SeqEntryAsnWrite (sep, aip, NULL);
            AsnIoFree (aip, FALSE);
          }
        }

        stoptime = GetSecs ();
        worsttime = stoptime - starttime;
        if (cfp->logfp != NULL) {
          fprintf (cfp->logfp, "ASN writing time %ld seconds\n", (long) worsttime);
          fflush (cfp->logfp);
        }
      }

      starttime = GetSecs ();

      for (x = 0; x < cfp->maxcount; x++) {
        DoProcess (sep, entityID, cfp);
      }

      stoptime = GetSecs ();
      worsttime = stoptime - starttime;
      if (cfp->logfp != NULL) {
        fprintf (cfp->logfp, "Internal processing time %ld seconds\n", (long) worsttime);
        fflush (cfp->logfp);
      }

      ObjMgrFreeByEntityID (entityID);

      bsplist = UnlockFarComponents (bsplist);
    }

  } else {

    Message (MSG_POSTERR, "Datatype %d not recognized", (int) datatype);
  }
}

static void ProcessMultipleRecord (
  CharPtr filename,
  CSpeedFlagPtr cfp
)

{
  AsnIoPtr     aip;
  AsnTypePtr   atp;
  BioseqPtr    bsp;
  Char         buf [41];
  Uint2        entityID;
  FILE         *fp;
  SeqEntryPtr  fsep;
  Char         longest [41];
  Int4         numrecords, x;
  SeqEntryPtr  sep;
  time_t       starttime, stoptime, worsttime;
#ifdef OS_UNIX
  Char         cmmd [256];
  CharPtr      gzcatprog;
  int          ret;
  Boolean      usedPopen = FALSE;
#endif

  if (cfp == NULL) return;

  if (StringHasNoText (filename)) return;

#ifndef OS_UNIX
  if (cfp->compressed) {
    Message (MSG_POSTERR, "Can only decompress on-the-fly on UNIX machines");
    return;
  }
#endif

#ifdef OS_UNIX
  if (cfp->compressed) {
    gzcatprog = getenv ("NCBI_UNCOMPRESS_BINARY");
    if (gzcatprog != NULL) {
      sprintf (cmmd, "%s %s", gzcatprog, filename);
    } else {
      ret = system ("gzcat -h >/dev/null 2>&1");
      if (ret == 0) {
        sprintf (cmmd, "gzcat %s", filename);
      } else if (ret == -1) {
        Message (MSG_POSTERR, "Unable to fork or exec gzcat in ScanBioseqSetRelease");
        return;
      } else {
        ret = system ("zcat -h >/dev/null 2>&1");
        if (ret == 0) {
          sprintf (cmmd, "zcat %s", filename);
        } else if (ret == -1) {
          Message (MSG_POSTERR, "Unable to fork or exec zcat in ScanBioseqSetRelease");
          return;
        } else {
          Message (MSG_POSTERR, "Unable to find zcat or gzcat in ScanBioseqSetRelease - please edit your PATH environment variable");
          return;
        }
      }
    }
    fp = popen (cmmd, /* cfp->binary? "rb" : */ "r");
    usedPopen = TRUE;
  } else {
    fp = FileOpen (filename, cfp->binary? "rb" : "r");
  }
#else
  fp = FileOpen (filename, cfp->binary? "rb" : "r");
#endif
  if (fp == NULL) {
    Message (MSG_POSTERR, "FileOpen failed for input file '%s'", filename);
    return;
  }

  aip = AsnIoNew (cfp->binary? ASNIO_BIN_IN : ASNIO_TEXT_IN, fp, NULL, NULL, NULL);
  if (aip == NULL) {
    Message (MSG_ERROR, "AsnIoNew failed for input file '%s'", filename);
    return;
  }

  if (cfp->logfp != NULL) {
    fprintf (cfp->logfp, "%s\n\n", filename);
    fflush (cfp->logfp);
  }

  longest [0] = '\0';
  worsttime = 0;
  numrecords = 0;

  atp = cfp->atp_bss;

  while ((atp = AsnReadId (aip, cfp->amp, atp)) != NULL) {
    if (atp == cfp->atp_se) {

      SeqMgrHoldIndexing (TRUE);
      sep = SeqEntryAsnRead (aip, atp);
      SeqMgrHoldIndexing (FALSE);

      if (sep != NULL) {

        entityID = ObjMgrGetEntityIDForChoice (sep);

        fsep = FindNthBioseq (sep, 1);
        if (fsep != NULL && fsep->choice == 1) {
          bsp = (BioseqPtr) fsep->data.ptrvalue;
          if (bsp != NULL) {
            SeqIdWrite (bsp->id, buf, PRINTID_FASTA_LONG, sizeof (buf));
            if (cfp->logfp != NULL) {
              fprintf (cfp->logfp, "%s\n", buf);
              fflush (cfp->logfp);
            }
          }
        }

        starttime = GetSecs ();

        for (x = 0; x < cfp->maxcount; x++) {
          DoProcess (sep, entityID, cfp);
        }
        stoptime = GetSecs ();

        if (stoptime - starttime > worsttime) {
          worsttime = stoptime - starttime;
          StringCpy (longest, buf);
        }
        numrecords++;

        ObjMgrFreeByEntityID (entityID);
      }

    } else {

      AsnReadVal (aip, atp, NULL);
    }
  }

  AsnIoFree (aip, FALSE);

#ifdef OS_UNIX
  if (usedPopen) {
    pclose (fp);
  } else {
    FileClose (fp);
  }
#else
  FileClose (fp);
#endif
  if (cfp->logfp != NULL && (! StringHasNoText (longest))) {
    fprintf (cfp->logfp, "Longest processing time %ld seconds on %s\n",
             (long) worsttime, longest);
    fprintf (cfp->logfp, "Total number of records %ld\n", (long) numrecords);
    fflush (cfp->logfp);
  }
}

static void ProcessOneRecord (
  CharPtr filename,
  Pointer userdata
)

{
  CSpeedFlagPtr  cfp;

  if (StringHasNoText (filename)) return;
  cfp = (CSpeedFlagPtr) userdata;
  if (cfp == NULL) return;

  if (cfp->batch) {
    ProcessMultipleRecord (filename, cfp);
  } else {
    ProcessSingleRecord (filename, cfp);
  }
}

/* Args structure contains command-line arguments */

#define p_argInputPath     0
#define i_argInputFile     1
#define o_argOutputFile    2
#define f_argFilter        3
#define x_argSuffix        4
#define a_argType          5
#define b_argBinary        6
#define c_argCompressed    7
#define l_argLockFar       8
#define L_argLogFile       9
#define R_argRemote       10
#define X_argMaxCount     11
#define O_argInOut        12
#define K_argClean        13
#define P_argSkip         14
#define I_argIndex        15
#define S_argSeq          16
#define F_argFeat         17
#define D_argDesc         18
#define V_argVerify       19

Args myargs [] = {
  {"Path to Files", NULL, NULL, NULL,
    TRUE, 'p', ARG_STRING, 0.0, 0, NULL},
  {"Single Input File", "stdin", NULL, NULL,
    TRUE, 'i', ARG_FILE_IN, 0.0, 0, NULL},
  {"Output File", "stdout", NULL, NULL,
    TRUE, 'o', ARG_FILE_OUT, 0.0, 0, NULL},
  {"Substring Filter", NULL, NULL, NULL,
    TRUE, 'f', ARG_STRING, 0.0, 0, NULL},
  {"File Selection Suffix", ".ent", NULL, NULL,
    TRUE, 'x', ARG_STRING, 0.0, 0, NULL},
  {"ASN.1 Type\n"
   "      a Any\n"
   "      e Seq-entry\n"
   "      b Bioseq\n"
   "      s Bioseq-set\n"
   "      m Seq-submit\n"
   "      t Batch Processing\n"
   "      f FASTA\n"
   "      l Read by Lines", "a", NULL, NULL,
    TRUE, 'a', ARG_STRING, 0.0, 0, NULL},
  {"Bioseq-set is Binary", "F", NULL, NULL,
    TRUE, 'b', ARG_BOOLEAN, 0.0, 0, NULL},
  {"Bioseq-set is Compressed", "F", NULL, NULL,
    TRUE, 'c', ARG_BOOLEAN, 0.0, 0, NULL},
  {"Lock Components in Advance", "F", NULL, NULL,
    TRUE, 'l', ARG_BOOLEAN, 0.0, 0, NULL},
  {"Log File", NULL, NULL, NULL,
    TRUE, 'L', ARG_FILE_OUT, 0.0, 0, NULL},
  {"Remote Fetching from ID", "F", NULL, NULL,
    TRUE, 'R', ARG_BOOLEAN, 0.0, 0, NULL},
  {"Max Repeat Count", "1", NULL, NULL,
    TRUE, 'X', ARG_INT, 0.0, 0, NULL},
  {"Input Output\n"
   "      r Read ASN.1\n"
   "      w Write Text ASN.1\n"
   "      wb Write Binary ASN.1", NULL, NULL, NULL,
    TRUE, 'O', ARG_STRING, 0.0, 0, NULL},
  {"Cleanup\n"
   "      m Mark Titles\n"
   "      d Delete Marked Objects\n"
   "      c Clear Feature Indexes\n"
   "      t Remove Titles\n"
   "      a AssignIDsInEntity\n"
   "      b BasicSeqEntryCleanup\n"
   "      s SeriousSeqEntryCleanup", NULL, NULL, NULL,
    TRUE, 'K', ARG_STRING, 0.0, 0, NULL},
  {"Skip\n"
   "      s Segmented Set Components\n"
   "      v Virtual Bioseqs", NULL, NULL, NULL,
    TRUE, 'P', ARG_STRING, 0.0, 0, NULL},
  {"Index\n"
   "      f Feature Indexing", NULL, NULL, NULL,
    TRUE, 'I', ARG_STRING, 0.0, 0, NULL},
  {"Sequence\n"
   "      c Compare FASTA Deflines\n"
   "      C Compare Regenerated FASTA Deflines\n"
   "      s FASTA of Sequence\n"
   "      S Indexed FASTA\n"
   "      r Raw FASTA no Defline\n"
   "      d Just FASTA Defline\n"
   "      D Indexed FASTA Defline\n"
   "      T Regenerate FASTA Titles\n"
   "      x New FASTA Titles\n"
   "      X Regenerate new FASTA Titles\n"
   "      f FASTA by Feature\n"
   "      t FASTA of Translation", NULL, NULL, NULL,
    TRUE, 'S', ARG_STRING, 0.0, 0, NULL},
  {"Feature\n"
   "      v Visit Features\n"
   "      g Gene Overlap Print\n"
   "      h Gene Overlap Speed\n"
   "      x Gene by Xref\n"
   "      o Operon by Overlap\n"
   "      d Feature by ID\n"
   "      t Feature Table\n"
   "      s Slow Suggest Intervals\n"
   "      S Indexed Suggest Intervals\n"
   "      c Coding Region Intervals", NULL, NULL, NULL,
    TRUE, 'F', ARG_STRING, 0.0, 0, NULL},
  {"Descriptor\n"
   "      b BioSource\n"
   "      t Title", NULL, NULL, NULL,
    TRUE, 'D', ARG_STRING, 0.0, 0, NULL},
  {"Verification\n"
   "      v Validate with Normal Stringency\n"
   "      b Generate GenBank Flatfile\n", NULL, NULL, NULL,
    TRUE, 'V', ARG_STRING, 0.0, 0, NULL},
};

Int2 Main (void)

{
  Char            app [64], type;
  CSpeedFlagData  cfd;
  CharPtr         directory, filter, infile, logfile, outfile, str, suffix;
  Boolean         remote;
  time_t          runtime, starttime, stoptime;

  /* standard setup */

  ErrSetFatalLevel (SEV_MAX);
  ErrClearOptFlags (EO_SHOW_USERSTR);
  UseLocalAsnloadDataAndErrMsg ();
  ErrPathReset ();

  /* finish resolving internal connections in ASN.1 parse tables */

  if (! AllObjLoad ()) {
    Message (MSG_FATAL, "AllObjLoad failed");
    return 1;
  }
  if (! SubmitAsnLoad ()) {
    Message (MSG_FATAL, "SubmitAsnLoad failed");
    return 1;
  }
  if (! FeatDefSetLoad ()) {
    Message (MSG_FATAL, "FeatDefSetLoad failed");
    return 1;
  }
  if (! SeqCodeSetLoad ()) {
    Message (MSG_FATAL, "SeqCodeSetLoad failed");
    return 1;
  }
  if (! GeneticCodeTableLoad ()) {
    Message (MSG_FATAL, "GeneticCodeTableLoad failed");
    return 1;
  }

  /* process command line arguments */

  sprintf (app, "cspeedtest %s", CSPEEDTEST_APPLICATION);
  if (! GetArgs (app, sizeof (myargs) / sizeof (Args), myargs)) {
    return 0;
  }

  MemSet ((Pointer) &cfd, 0, sizeof (CSpeedFlagData));

  directory = (CharPtr) myargs [p_argInputPath].strvalue;
  infile = (CharPtr) myargs [i_argInputFile].strvalue;
  outfile = (CharPtr) myargs [o_argOutputFile].strvalue;
  filter = (CharPtr) myargs [f_argFilter].strvalue;
  suffix = (CharPtr) myargs [x_argSuffix].strvalue;

  cfd.batch = FALSE;
  cfd.binary = (Boolean) myargs [b_argBinary].intvalue;
  cfd.compressed = (Boolean) myargs [c_argCompressed].intvalue;
  cfd.lock = (Boolean) myargs [l_argLockFar].intvalue;
  cfd.type = 1;

  str = myargs [a_argType].strvalue;
  TrimSpacesAroundString (str);
  if (StringDoesHaveText (str)) {
    type = str [0];
  } else {
    type = 'a';
  }

  type = TO_LOWER (type);
  switch (type) {
    case 'a' :
      cfd.type = 1;
      break;
    case 'e' :
      cfd.type = 2;
      break;
    case 'b' :
      cfd.type = 3;
      break;
    case 's' :
      cfd.type = 4;
      break;
    case 'm' :
      cfd.type = 5;
      break;
    case 't' :
      cfd.type = 1;
      cfd.batch = TRUE;
      break;
    case 'f' :
      cfd.type = 6;
      break;
    case 'l' :
      cfd.type = 7;
      break;
    default :
      cfd.type = 1;
      break;
  }

  remote = (Boolean) myargs [R_argRemote].intvalue;

  cfd.maxcount = myargs [X_argMaxCount].intvalue;
  if (cfd.maxcount < 1) {
    cfd.maxcount = 1;
  }

  cfd.io = myargs [O_argInOut].strvalue;
  cfd.clean = myargs [K_argClean].strvalue;
  cfd.skip = myargs [P_argSkip].strvalue;
  cfd.index = myargs [I_argIndex].strvalue;
  cfd.seq = myargs [S_argSeq].strvalue;
  cfd.feat = myargs [F_argFeat].strvalue;
  cfd.desc = myargs [D_argDesc].strvalue;
  cfd.verify = myargs [V_argVerify].strvalue;

  cfd.amp = AsnAllModPtr ();
  cfd.atp_bss = AsnFind ("Bioseq-set");
  cfd.atp_bsss = AsnFind ("Bioseq-set.seq-set");
  cfd.atp_se = AsnFind ("Bioseq-set.seq-set.E");
  cfd.atp_bsc = AsnFind ("Bioseq-set.class");
  cfd.bssp_atp = AsnLinkType (NULL, cfd.atp_bss);

  logfile = (CharPtr) myargs [L_argLogFile].strvalue;
  if (StringDoesHaveText (logfile)) {
    cfd.logfp = FileOpen (logfile, "w");
  }

  if (remote) {
    PubSeqFetchEnable ();
  }

  if (StringDoesHaveText (outfile)) {
    cfd.ofp = FileOpen (outfile, "w");
  }

  starttime = GetSecs ();

  if (StringDoesHaveText (directory)) {

    DirExplore (directory, NULL, suffix, FALSE, ProcessOneRecord, (Pointer) &cfd);

  } else if (StringDoesHaveText (infile)) {

    ProcessOneRecord (infile, (Pointer) &cfd);
  }

  if (cfd.ofp != NULL) {
    FileClose (cfd.ofp);
  }

  stoptime = GetSecs ();
  runtime = stoptime - starttime;
  if (cfd.logfp != NULL) {
    fprintf (cfd.logfp, "Finished in %ld seconds\n", (long) runtime);
    FileClose (cfd.logfp);
  }
  printf ("Finished in %ld seconds\n", (long) runtime);

  if (remote) {
    PubSeqFetchDisable ();
  }

  return 0;
}