/*
//
// htp.c
//
// main(), major functionality modules
//
// Copyright (c) 1995-96 Jim Nelson.  Permission to distribute
// granted by the author.  No warranties are made on the fitness of this
// source code.
//
*/

#include "htp.h"

/*
// default response filename
*/
const char *DEFAULT_RESPONSE_FILE = "htp.rsp";

/*
// variable types
*/
#define VAR_TYPE_SET_MACRO      (1)
#define VAR_TYPE_BLOCK_MACRO    (2)
#define VAR_TYPE_INTERNAL       (3)
#define VAR_TYPE_ALTTEXT        (4)
#define VAR_TYPE_DEF_MACRO      (5)

/*
// variable flags
*/
#define VAR_FLAG_NONE           (0x0000)
#define VAR_FLAG_QUOTED         (0x0001)

/*
// specialized markup processors type definitions
*/

/* MARKUP_FUNC return codes */
#define MARKUP_OKAY             (0)
#define MARKUP_REPLACED         (1)
#define DISCARD_MARKUP          (2)
#define MARKUP_ERROR            ((uint) -1)
#define NEW_MARKUP              (3)

/*
// when reading source files, need to dynamically allocate memory to avoid
// overruns ... use a stronger strategy for "real" operating systems, more
// conservative for wimpy DOS
*/
#if __MSDOS__

#define MIN_PLAINTEXT_SIZE      (128)
#define PLAINTEXT_GROW_SIZE     (32)

#else

#define MIN_PLAINTEXT_SIZE      (16 * KBYTE)
#define PLAINTEXT_GROW_SIZE     (4 * KBYTE)

#endif


/*
// miscellaneous definitions
*/
#define MAX_TIME_DATE_SIZE      (128)
#define DEFAULT_PRECISION       (0)
#define SEARCH_PATH_SIZE        (1024)

/*
// htp task structure
//
// (the word "task" is not to be confused with the traditional operating system
// term ... it is used here to represent all the information associated with
// the particular job at hand, which is reading in a file, writing out to
// a file, and maintaining information during the entire operation)
*/
typedef struct tagTASK
{
    TEXTFILE        *infile;
    TEXTFILE        *outfile;
    VARSTORE        *varstore;
    const char      *sourceFilename;
} TASK;

/*
// markup processor function and array association structure
*/
typedef uint (*MARKUP_FUNC)(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext);
typedef struct tagMARKUP_PROCESSORS
{
    const char      *tag;
    uint            markupType;
    MARKUP_FUNC     markupFunc;
} MARKUP_PROCESSORS;

/*
// template file name (used internally to store name for post-processing)
// use squirrelly characters and a space to avoid conflicting with
// user names
*/
const char *VAR_TEMPLATE_NAME = "__!TEMPLATE FILE~";

/*
// forward references
*/
BOOL ProcessTask(TASK *task);
BOOL OptionCallback(const char *name, const char *value, ulong userParam);

/*
// the user can configure what kind of characters to use to surround htp
// markups, to avoid conflicts with HTML markups ... default is the standard
// greater-than/less-than bracketing, but also acceptable are square
// brackets and curly brackets (parentheses are just too common in normal
// text to be useful)
//
// Because htp also processes standard HTML markups, a IS_OPEN_MARKUP and
// IS_CLOSE_MARKUP macros are used instead of standard comparisons ... watch
// out for side-effects
//
// MARKUP_TYPE_ANY is used for markup processors to define they are
// interested in either kind of markup (currently unused)
//
// MARKUP_OPEN_DELIM and MARKUP_CLOSE_DELIM are used to return the proper
// delimiter given the markup type
*/
#define HTML_OPEN_MARKUP            ('<')
#define HTML_CLOSE_MARKUP           ('>')

char htpOpenMarkup = HTML_OPEN_MARKUP;
char htpCloseMarkup = HTML_CLOSE_MARKUP;

#define IS_OPEN_MARKUP(c)           (((c) == '<') || ((c) == htpOpenMarkup))
#define IS_CLOSE_MARKUP(c)          (((c) == '>') || ((c) == htpCloseMarkup))

#define MARKUP_TYPE_HTML            (0x0001)
#define MARKUP_TYPE_HTP             (0x0002)
#define MARKUP_TYPE_ANY             (0xFFFF)

#define MARKUP_OPEN_DELIM(t) \
    (((t) & MARKUP_TYPE_HTP) ? htpOpenMarkup : HTML_OPEN_MARKUP)

#define MARKUP_CLOSE_DELIM(t) \
    (((t) & MARKUP_TYPE_HTP) ? htpCloseMarkup : HTML_CLOSE_MARKUP)

/*
// the global variable store ... holds permanent, file-to-file macros
// (these are set in the global default file) ... filename kept for
// dependency checking
*/
VARSTORE globalVarStore;
char globalFilename[MAX_PATHNAME_LEN];

/*
// the "project" variable store ... holds macros only for files in current
// directory, or project (this is loaded from the project default file,
// which is called htp.def)
*/
VARSTORE projectVarStore;
char projectFilename[MAX_PATHNAME_LEN];

/*
// include file search path
*/
char searchPath[SEARCH_PATH_SIZE] = { 0, };

/*
// ALT text macro store
*/
VARSTORE altTextVarStore;

/*
// for tracking ExpandMacros performance, debug version only
*/
#if DEBUG
uint expandSkipped = 0;
uint expandPerformed = 0;
#endif

/*
//
// generic, global utility functions
//
*/

/*
// temporary file name generator
*/

BOOL CreateTempFilename(char *tempfilename, uint size)
{
    static char *tempDir;
    char *tmpName;

    assert(tempfilename != NULL);

    /* find a preferred temporary directory if not found already */
    if(tempDir == NULL)
    {
        if((tempDir = getenv("TEMP")) == NULL)
        {
            if((tempDir = getenv("TMP")) == NULL)
            {
                tempDir = DIR_CURRENT_STRING;
            }
        }
    }

    /* get a temporary filename */
    if((tmpName = tempnam(tempDir, (char *) PROGRAM_NAME)) != NULL)
    {
        /* copy the filename to the callers buffer */
        StringCopy(tempfilename, tmpName, size);

        /* free the tempnam buffer and return success */
        free(tmpName);

        return TRUE;
    }

    return FALSE;
}


#if 0

/*
// extract directory from a filename, if present
*/

char *GetFileDirectory(const char *filename, char *directory, uint size)
{
    const char *filePtr;
    uint len;

    *directory = NUL;

    len = strlen(filename);
    if(len == 0)
    {
        return directory;
    }

    filePtr = filename + len - 1;
    while(filePtr != filename)
    {
        if(*filePtr == DIR_DELIMITER)
        {
            return StringCopy(directory, filename, (len <= size) ? len : size);
        }

        filePtr--;
        len--;
    }

    return directory;
}   

#endif

/*
// ParseFilename
//
// Returns a pointer to the filename in a full pathname
*/
char *FindFilename(char *pathname)
{
    char *filePtr;
    uint len;

    assert(pathname != NULL);
    if(pathname == NULL)
    {
        return NULL;
    }

    len = strlen(pathname);
    if(len == 0)
    {
        return pathname;
    }

    filePtr = pathname + len - 1;
    while(filePtr != pathname)
    {
        if(strchr(ALL_FILESYSTEM_DELIMITERS, *filePtr) != NULL)
        {
            /* found the first delimiter, return pointer to character just */
            /* past this one */
            /* if pathname ended in a delimiter, then this will return a */
            /* pointer to NUL, which is acceptable */
            return filePtr + 1;
        }

        filePtr--;
    }

    return pathname;
}

/*
// safe strncpy() wrapper (suggested by joseph.dandrea@att.com) ... strncpy()
// by itself has some ugly problems, and strcpy() is simply dangerous.
// Joseph recommended a macro, but I'm sticking to the bulkier solution of
// using a function
*/
char *StringCopy(char *dest, const char *src, uint size)
{
    assert(dest != NULL);
    assert(src != NULL);

    strncpy(dest, src, size);
    dest[size - 1] = NUL;

    return dest;
}

/*
// re-entrant string tokenizer ... used because option.c requires simultaneous
// uses of strtok(), and the standard version just dont cut it
*/
char *StringFirstToken(FIND_TOKEN *findToken, char *string, const char *tokens)
{
    char *ptr;

    assert(string != NULL);
    assert(findToken != NULL);

    findToken->tokens = tokens;
    findToken->lastChar = string + strlen(string);
    findToken->nextStart = findToken->lastChar;

    if(tokens == NULL)
    {
        return string;
    }

    if((ptr = strpbrk(string, tokens)) != NULL)
    {
        *ptr = NUL;
        findToken->nextStart = ptr;
    }

    return string;
}

char *StringNextToken(FIND_TOKEN *findToken)
{
    char *ptr;
    char *start;

    assert(findToken != NULL);
    assert(findToken->lastChar != NULL);

    ptr = findToken->nextStart;

    /* check if this is the end of the original string */
    if(ptr == findToken->lastChar)
    {
        return NULL;
    }

    /* nextStart points to NUL left by last search, skip past it */
    ptr++;
    start = ptr;

    /* keep going */
    if((ptr = strpbrk(ptr, findToken->tokens)) != NULL)
    {
        *ptr = NUL;
        findToken->nextStart = ptr;
    }
    else
    {
        findToken->nextStart = findToken->lastChar;
    }

    return start;
}

/*
// Wrapper function to (a) allocate memory for the duplicated string and
// (b) copy the source string into the new memory location.  Caller is
// responsible to free the string eventually.
*/
char *DuplicateString(const char *src)
{
    char *new;
    uint size;

    assert(src != NULL);

    size = strlen(src) + 1;

    /* allocate memory for the duplicate string */
    if((new = AllocMemory(size)) == NULL)
    {
        return NULL;
    }

    /* copy the string */
    return memcpy(new, src, size);
}

/*
// returns the full, qualified pathname of the default htp include file
//
// Returns FALSE if unable to find the file.
*/
BOOL HtpDefaultFilename(char *filename, uint size)
{
    char *defFile;

    /* get the name of the default file from the HTPDEF environement */
    /* variable */
    if((defFile = getenv("HTPDEF")) == NULL)
    {
        return FALSE;
    }

    /* verify that the file exists */
    if(FileExists(defFile) == FALSE)
    {
        return FALSE;
    }

    /* copy the filename into the buffer and skeedaddle */
    StringCopy(filename, defFile, size);

    return TRUE;
}

/*
// compare files modified time/date stamp, as a dependency check ... returns
// TRUE if the dependency does not require an update, FALSE otherwise (which
// could either be a timestamp discrepency, or simply that the resulting file
// does not exist) ... if dependency checking is turned off, this function
// will always return FALSE.
//
// Returns ERROR if dependency file does not exist.
*/
BOOL IsTargetUpdated(const char *dependency, const char *target)
{
    struct stat dependStat;
    struct stat targetStat;
    char *dependName;
    char *targetName;

    assert(dependency != NULL);
    assert(target != NULL);

    /* always update targets? */
    if(DEPEND == FALSE)
    {
        return FALSE;
    }

    /* convert the dependency and target filenames for this filesystem */
    if((dependName = ConvertDirDelimiter(dependency)) == NULL)
    {
        return ERROR;
    }

    if((targetName = ConvertDirDelimiter(target)) == NULL)
    {
        FreeMemory(dependName);
        return ERROR;
    }

    /* get information on the dependency file */
    if(stat(dependName, &dependStat) != 0)
    {
        /* dependency file needs to exist */
        FreeMemory(dependName);
        FreeMemory(targetName);

        return ERROR;
    }

    /* get information on the target file */
    if(stat(targetName, &targetStat) != 0)
    {
        /* target file does not exist, dependency needs to be updated */
        FreeMemory(dependName);
        FreeMemory(targetName);

        return FALSE;
    }

    FreeMemory(dependName);
    FreeMemory(targetName);

    /* compare modification times to determine if up-to-date */
    return (dependStat.st_mtime <= targetStat.st_mtime) ? TRUE : FALSE;
}

/*
// converts directory delimiters for any pathname into one supporting the
// delimiters used by the present filesystem ... it is encumbent on the
// caller to free() the string returned once finished
*/
char *ConvertDirDelimiter(const char *pathname)
{
    char *newPathname;
    char *strptr;

    if(pathname == NULL)
    {
        return NULL;
    }

    /* duplicate the pathname for conversion */
    if((newPathname = DuplicateString(pathname)) == NULL)
    {
        return NULL;
    }

    /* walk the string, looking for delimiters belonging to other filesystems */
    /* replace with native filesystems delimiter */
    strptr = newPathname;
    while(*strptr != NUL)
    {
        if(strchr(OTHER_FILESYSTEM_DELIMITER, *strptr) != NULL)
        {
            *strptr = DIR_DELIMITER;
        }
        strptr++;
    }

    return newPathname;
}

/*
// uses stat() to check for file existance ... maybe not the best way?
*/
BOOL FileExists(const char *pathname)
{
    struct stat dummy;

    return (stat(pathname, &dummy) == 0) ? TRUE : FALSE;
}

/*
// searches for the specified file in the search path ... this function is
// very stupid, it simply gets the first directory in the search string,
// appends the file directly to the end, and tests for existance.  Repeat.
*/
BOOL SearchForFile(const char *filename, char *fullPathname, uint size)
{
    char *searchPathCopy;
    char *ptr;
    char *convertedName;
    FIND_TOKEN findToken;

    /* quick check for search path even being defined */
    if(searchPath[0] == NUL)
    {
        return FALSE;
    }

    /* need to make a copy of the search path for String...Token() to butcher up */
    if((searchPathCopy = DuplicateString(searchPath)) == NULL)
    {
        printf("%s: unable to allocate temporary buffer for include path (out of memory?)\n",
            PROGRAM_NAME);
        return FALSE;
    }

    /* look for ';' delimiter */
    ptr = StringFirstToken(&findToken, searchPathCopy, ";");
    while(ptr != NULL)
    {
        StringCopy(fullPathname, ptr, size);

        /* if the last character is not a directory delimiter, add it */
        if(strchr(ALL_FILESYSTEM_DELIMITERS, fullPathname[strlen(fullPathname) - 1]) == NULL)
        {
            strncat(fullPathname, DIR_DELIMITER_STRING, size);
        }

        /* append the file name */
        strncat(fullPathname, filename, size);

        /* need to do a complete conversion of delimiters in the filename, but */
        /* ConvertDirDelimiter() returns a AllocMemory()'d copy of the string ... */
        convertedName = ConvertDirDelimiter(fullPathname);

        /* check for existance */
        if(FileExists(convertedName) == TRUE)
        {
            /* clean up and get outta here */
            StringCopy(fullPathname, convertedName, size);

            FreeMemory(searchPathCopy);
            FreeMemory(convertedName);

            return TRUE;
        }

        FreeMemory(convertedName);
        convertedName = NULL;

        ptr = StringNextToken(&findToken);
    }

    /* clean up */
    FreeMemory(searchPathCopy);

    return FALSE;
}

/*
// HTML file stream functions
*/

/*
// returns the markup type flag ... either the closing or opening delimiter
// in the markup can be passed in
*/
uint MarkupType(char delim)
{
    uint markupType;

    markupType = 0;

    if((delim == HTML_OPEN_MARKUP) || (delim == HTML_CLOSE_MARKUP))
    {
        markupType |= MARKUP_TYPE_HTML;
    }

    if((delim == htpOpenMarkup) || (delim == htpCloseMarkup))
    {
        markupType |= MARKUP_TYPE_HTP;
    }

    return markupType;
}

/*
// TRUE = plaintext is filled with new plain text markup, FALSE if end of file,
// ERROR if a problem
// !! Don't like using ERROR in any BOOL return values
*/
BOOL ReadHtmlFile(TEXTFILE *infile, TEXTFILE *outfile, char **plaintext,
    uint *markupType)
{
    char ch;
    uint ctr;
    char *buffer;
    char *newbuffer;
    uint size;
    uint bytesReadPrevLine;
    BOOL inQuotes;
    uint startLine;

    assert(infile != NULL);
    assert(plaintext != NULL);

    /* if outfile is NULL, then the input stream is just being walked and */
    /* not parsed for output ... i.e., don't assert outfile != NULL */

    /* allocate some space for markup plaintext ... this will dynamically */
    /* expand if necessary, and has to be freed by the caller */
    if((buffer = AllocMemory(MIN_PLAINTEXT_SIZE)) == NULL)
    {
        HtpMsg(MSG_ERROR, NULL, "unable to allocate memory to read HTML file");
        return ERROR;
    }

    /* track the buffer size */
    size = MIN_PLAINTEXT_SIZE;

    for(;;)
    {
        /* GetFileChar() will reset bytesReadThisLine if a EOL char is read */
        /* so save it for later evaluation ... (bytesReadPrevLine is named */
        /* to indicate it is only evaluated if a EOL char is read) */
        bytesReadPrevLine = infile->bytesReadThisLine;
        if(GetFileChar(infile, &ch) == FALSE)
        {
            break;
        }

        if(IS_OPEN_MARKUP(ch) == FALSE)
        {
            /* normal text, just copy it to the output file (if there is one) */
            if(outfile != NULL)
            {
                /* this is used to catch spurious EOLs sneaking into the final output */
                /* essentially, if bytes were read but none were written, it */
                /* is assumed that the only text on the line were htp directives */
                /* and therefore it is unnecessary (and ugly) to write out the */
                /* EOL at the end of the line */
                if(ch == '\n')
                {
                    if((bytesReadPrevLine > 0) && (outfile->bytesWrittenThisLine == 0))
                    {
                        continue;
                    }
                }

                PutFileChar(outfile, ch);
            }
        }
        else
        {
            /* get the type of markup for caller */
            *markupType = MarkupType(ch);

            /* copy the markup into the buffer */
            ctr = 0;
            inQuotes = FALSE;
            startLine = infile->lineNumber;
            for(;;)
            {
                if(GetFileChar(infile, &ch) == FALSE)
                {
                    /* EOF ... this is not acceptable before the markup is */
                    /* terminated */
                    FreeMemory(buffer);
                    HtpMsg(MSG_ERROR, infile, "EOF encountered inside markup tag (started on line %u)",
                        startLine);

                    return ERROR;
                }

                if((IS_CLOSE_MARKUP(ch))
                    && (inQuotes == FALSE)
                    && (MarkupType(ch) == *markupType))
                {
                    /* end of markup, terminate string and exit */
                    buffer[ctr] = NUL;
                    break;
                }

                /* track quotation marks ... can only close markup when */
                /* all quotes have been closed */
                if(ch == '\"')
                {
                    inQuotes = (inQuotes == TRUE) ? FALSE : TRUE;
                }

                /* copy it into the plaintext buffer */
                buffer[ctr++] = ch;

                /* check for overflow ... resize buffer if necessary */
                if(ctr == size)
                {
                    newbuffer = ResizeMemory(buffer, size + PLAINTEXT_GROW_SIZE);
                    if(newbuffer == NULL)
                    {
                        /* unable to enlarge buffer area */
                        HtpMsg(MSG_ERROR, NULL, "unable to reallocate memory for reading HTML file");
                        FreeMemory(buffer);
                        return ERROR;
                    }

                    buffer = newbuffer;
                    size += PLAINTEXT_GROW_SIZE;
                }
            }

            /* give the buffer pointer to the caller */
            *plaintext = buffer;

            return TRUE;
        }
    }

    /* no markup found, end of file */
    FreeMemory(buffer);

    return FALSE;
}   

/*
// specialized markup processors
*/

uint ImageProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    const char *imgfilename;
    char str[32];
    IMAGEFILE imageFile;
    WORD width, height;
    char *imgSource;
    const char *originalSource;
    char *imgFilename;
    const char *imgText;

    UNREF_PARAM(newPlaintext);

    /* try to find ALT text in store */
    /* first: is there already ALT attribute?  if so, skip this step */
    if(IsAttributeInMarkup(htmlMarkup, "ALT") == FALSE)
    {
        /* parse down the image source to just the filename */
        originalSource = MarkupAttributeValue(htmlMarkup, "SRC");
        if(originalSource == NULL)
        {
            HtpMsg(MSG_WARNING, task->infile, "image SRC not specified, skipping");
            return MARKUP_OKAY;
        }

        imgSource = DuplicateString(originalSource);
        if(imgSource == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "out of memory processing IMG tag");
            return MARKUP_ERROR;
        }

        /* parse the image name, just find the filename */
        imgFilename = FindFilename(imgSource);

        /* is the image in the ALT text store? */
        if(VariableExists(&altTextVarStore, imgFilename) == TRUE)
        {
            /* add the specified text to the image */
            imgText = GetVariableValue(&altTextVarStore, imgFilename);
            assert(imgText != NULL);
            AddAttributeToMarkup(htmlMarkup, "ALT", imgText, TRUE);
            HtpMsg(MSG_INFO, task->infile, "ALT text \"%s\" added to IMG \"%s\"",
                imgText, imgSource);
        }

        FreeMemory(imgSource);
        imgSource = NULL;
    }

    /* if option is turned off, then just include the markup as-is */
    if(IMGXY == FALSE)
    {
        return MARKUP_OKAY;
    }

    /* if width and/or height are already specified, then include the */
    /* markup as-is with no modifications */
    if(IsAttributeInMarkup(htmlMarkup, "HEIGHT")
        || IsAttributeInMarkup(htmlMarkup, "WIDTH"))
    {
        return MARKUP_OKAY;
    }

    /* get the filename of the image */
    if((imgfilename = MarkupAttributeValue(htmlMarkup, "SRC")) == NULL)
    {
        HtpMsg(MSG_ERROR, task->infile, "unable to retrieve image source name");
        return MARKUP_ERROR;
    }

    /* open the image file, get its dimensions, and close the file */
    if(OpenImageFile(imgfilename, &imageFile) == FALSE)
    {
        HtpMsg(MSG_WARNING, task->infile, "unable to open image file \"%s\"",
            imgfilename);
        return MARKUP_OKAY;
    }

    if(GetImageDimensions(&imageFile, &width, &height) == FALSE)
    {
        HtpMsg(MSG_WARNING, task->infile, "unable to determine image file \"%s\" dimensions",
            imgfilename);
        CloseImageFile(&imageFile);
        return MARKUP_OKAY;
    }

    CloseImageFile(&imageFile);

    /* add the width and height specifier for the image */
    sprintf(str, "%u", width);
    AddAttributeToMarkup(htmlMarkup, "WIDTH", str, TRUE);
    sprintf(str, "%u", height);
    AddAttributeToMarkup(htmlMarkup, "HEIGHT", str, TRUE);

    /* print out an informational message to the user */
    HtpMsg(MSG_INFO, task->outfile, "image file \"%s\" dimensions (%u x %u) added",
        imgfilename, width, height);

    /* include the markup in the final output */
    return MARKUP_OKAY;
}   

BOOL OptionCallback(const char *name, const char *value, ulong userParam)
{
    TASK *task;
    BOOL printMsg;

    task = (TASK *) userParam;
    printMsg = (task == NULL) ? FALSE : TRUE;

    if(name == NULL)
    {
        /* dont like it, but dont stop processing either */
        HtpMsg(MSG_WARNING, task->infile, "unknown option \"%s\" specified", value);
        return TRUE;
    }

    if(stricmp(name, OPT_N_QUIET) == 0)
    {
        if(stricmp(value, OPT_V_TRUE) == 0)
        {
            SetMessageSeverityLevel(MSG_WARNING);
        }
        else
        {
            SetMessageSeverityLevel(MSG_INFO);
        }
    }

    if(stricmp(name, OPT_N_IMGXY) == 0)
    {
        if(stricmp(value, OPT_V_TRUE) == 0)
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "image pre-processing turned ON");
            }
        }
        else
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "image pre-processing turned OFF");
            }
        }
    }

    if(stricmp(name, OPT_N_DEPEND) == 0)
    {
        if(stricmp(value, OPT_V_TRUE) == 0)
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "dependency checking turned ON");
            }
        }
        else
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "dependency checking turned OFF");
            }
        }
    }

    if(stricmp(name, OPT_N_PRECIOUS) == 0)
    {
        if(stricmp(value, OPT_V_TRUE) == 0)
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "precious output turned ON");
            }
        }
        else
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "precious output turned OFF");
            }
        }
    }

    if(stricmp(name, OPT_N_CONDENSE) == 0)
    {
        if(stricmp(value, OPT_V_TRUE) == 0)
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "output will be condensed");
            }

            if(task != NULL)
            {
                SuppressLinefeeds(task->outfile);
            }
        }
        else
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "output will not be condensed");
            }

            if(task != NULL)
            {
                AllowLinefeeds(task->outfile);
            }
        }
    }

    if(stricmp(name, OPT_N_KEEP_TEMP) == 0)
    {
        if(stricmp(value, OPT_V_TRUE) == 0)
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "DEBUG: keeping temporary files");
            }
        }
        else
        {
            if(printMsg)
            {
                HtpMsg(MSG_INFO, task->infile, "DEBUG: not keeping temporary files");
            }
        }
    }

    if(stricmp(name, OPT_N_SET_DELIM) == 0)
    {
        if(stricmp(value, OPT_V_DELIM_HTML) == 0)
        {
            htpOpenMarkup = '<';
            htpCloseMarkup = '>';
        }
        else if(stricmp(value, OPT_V_DELIM_SQUARE) == 0)
        {
            htpOpenMarkup = '[';
            htpCloseMarkup = ']';
        }
        else if(stricmp(value, OPT_V_DELIM_CURLY) == 0)
        {
            htpOpenMarkup = '{';
            htpCloseMarkup = '}';
        }

        if(printMsg)
        {
            HtpMsg(MSG_INFO, task->infile, "markup delimiter set to \"%s\"",
                    value);
        }
    }

    return TRUE;
}

uint OptionProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    UNREF_PARAM(newPlaintext);

    if(ParseMarkupOption(htmlMarkup, OptionCallback, (ulong) task) == FALSE)
    {
        HtpMsg(MSG_ERROR, task->infile, "error parsing markup options (out of memory?)");
        return MARKUP_ERROR;
    }

    return DISCARD_MARKUP;
}   

uint ExternalFileProcessor(TASK *task, HTML_MARKUP *htmlMarkup,
    const char *externalName, char **newPlaintext)
{
    struct stat fileStat;
    struct tm *fileTime;
    uint precision;
    const char *attribValue;
    const char *precisionString;
    const char *value;

    assert(externalName != NULL);
    assert(htmlMarkup != NULL);

    /* get information on the file itself */
    if(stat(externalName, &fileStat) != 0)
    {
        HtpMsg(MSG_ERROR, task->infile, "unable to retrieve file information on \"%s\"",
            externalName);
        return MARKUP_ERROR;
    }

    /* get the precision attribute value, if present */
    /* (this is only valid for SIZE attribute, but not checking for simplicity */
    /* ignored for other types of FILE attributes) */
    precision = DEFAULT_PRECISION;
    if(IsAttributeInMarkup(htmlMarkup, "PRECISION"))
    {
        precisionString = MarkupAttributeValue(htmlMarkup, "PRECISION");
        if(precisionString != NULL)
        {
            precision = atoi(precisionString);
        }
        else
        {
            HtpMsg(MSG_WARNING, task->infile, "precision attribute needs a value");
        }
    }

    /* allocate room for the replacment plaintext */
    if((*newPlaintext = AllocMemory(MAX_TIME_DATE_SIZE)) == NULL)
    {
        HtpMsg(MSG_ERROR, task->infile, "unable to allocate memory for expansion");
        return MARKUP_ERROR;
    }

    /* create new plaintext depending on what extra information is specified */
    /* !! this is technically not correct ... SIZE, TIME, and DATE are */
    /* not allowed in the same markup but not checking that only one is */
    /* present and not any others */
    if(IsAttributeInMarkup(htmlMarkup, "SIZE"))
    {
        attribValue = MarkupAttributeValue(htmlMarkup, "SIZE");

        /* expand markup depending on how SIZE should be represented */
        if((attribValue == NULL) || (stricmp(attribValue, "BYTE") == 0))
        {
            /* byte representation is default */
            sprintf(*newPlaintext, "%lu", fileStat.st_size);
        }
        else if(stricmp(attribValue, "KBYTE") == 0)
        {
            sprintf(*newPlaintext, "%.*f", (int) precision,
                (double) ((double) fileStat.st_size / (double) KBYTE));
        }
        else if(stricmp(attribValue, "MBYTE") == 0)
        {
            sprintf(*newPlaintext, "%.*f", (int) precision,
                (double) ((double) fileStat.st_size / (double) MBYTE));
        }
        else if(stricmp(attribValue, "GBYTE") == 0)
        {
            sprintf(*newPlaintext, "%.*f", (int) precision,
                (double) ((double) fileStat.st_size / (double) GBYTE));
        }
        else
        {
            /* free the plaintext memory before returning */
            HtpMsg(MSG_ERROR, task->infile, "unknown SIZE specifier");
            FreeMemory(*newPlaintext);
            *newPlaintext = NULL;
            return MARKUP_ERROR;
        }
    }
    else if(IsAttributeInMarkup(htmlMarkup, "TIME"))
    {
        const char *value;

        /* convert into an ANSI time structure */
        fileTime = localtime(&fileStat.st_mtime);

        /* see if the attribute has a value ... if so, let it be the */
        /* strftime() formatter */
        if((value = MarkupAttributeValue(htmlMarkup, "TIME")) != NULL)
        {
            strftime(*newPlaintext, MAX_TIME_DATE_SIZE, value, fileTime);
        }
        else
        {
            strftime(*newPlaintext, MAX_TIME_DATE_SIZE, "%I:%M:%S %p", fileTime);
        }
    }
    else if(IsAttributeInMarkup(htmlMarkup, "DATE"))
    {
        /* convert into an ANSI time structure */
        fileTime = localtime(&fileStat.st_mtime);

        /* see if the attribute has a value ... if so, let it be the */
        /* strftime() formatter */
        if((value = MarkupAttributeValue(htmlMarkup, "DATE")) != NULL)
        {
            strftime(*newPlaintext, MAX_TIME_DATE_SIZE, value, fileTime);
        }
        else
        {
            strftime(*newPlaintext, MAX_TIME_DATE_SIZE, "%a %b %d, %Y", fileTime);
        }
    }
    else
    {
        /* free the plaintext, unused */
        FreeMemory(*newPlaintext);
        *newPlaintext = NULL;

        HtpMsg(MSG_ERROR, task->infile, "bad file information specifier");
        return MARKUP_ERROR;
    }

    /* the new plaintext was created successfully */
    return NEW_MARKUP;
}

uint FileExecuteProcessor(TASK *task, HTML_MARKUP *htmlMarkup)
{
    const char *cmdline;
    const char *output;
    char newCmdline[MAX_CMDLINE_LEN];
    char tempFilename[MAX_PATHNAME_LEN];
    BOOL redirect;
    TEXTFILE infile;
    TASK newTask;
    BOOL result;
    uint exitCode;

    assert(task != NULL);
    assert(htmlMarkup != NULL);

    if((cmdline = MarkupAttributeValue(htmlMarkup, "EXECUTE")) == NULL)
    {
        HtpMsg(MSG_ERROR, task->infile,"FILE EXECUTE must specify a command-line");
        return MARKUP_ERROR;
    }

    output = MarkupAttributeValue(htmlMarkup, "OUTPUT");
    redirect = IsAttributeInMarkup(htmlMarkup, "REDIRECT");

    /* either output or redirect, but not both, must be specified */
    if((output == NULL) && (redirect == FALSE))
    {
        HtpMsg(MSG_ERROR, task->infile, "Either REDIRECT or OUTPUT must be specified for FILE EXECUTE");
        return MARKUP_ERROR;
    }

    if((output != NULL) && (redirect == TRUE))
    {
        HtpMsg(MSG_ERROR, task->infile, "REDIRECT and OUTPUT cannot both be specified for FILE EXECUTE");
        return MARKUP_ERROR;
    }

    StringCopy(newCmdline, cmdline, MAX_CMDLINE_LEN);

    /* if redirection required, append to the command-line a redirector to */
    /* a temporary file */
    if(redirect)
    {
        if(CreateTempFilename(tempFilename, MAX_PATHNAME_LEN) == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to create a temporary file for redirection");
            return MARKUP_ERROR;
        }

        strncat(newCmdline, " > ", MAX_PATHNAME_LEN);
        strncat(newCmdline, tempFilename, MAX_PATHNAME_LEN);
    }
    else
    {
        /* the specified output file is the "temporary" filename */
        StringCopy(tempFilename, output, MAX_PATHNAME_LEN);
    }

    HtpMsg(MSG_INFO, task->infile, "Executing command \"%s\" ...", newCmdline);

    /* execute the command */
    exitCode = system(newCmdline);

    if(exitCode != 0)
    {
        if(IsAttributeInMarkup(htmlMarkup, "NOERROR") == FALSE)
        {
            /* the program has exited with an error condition */
            HtpMsg(MSG_ERROR, task->infile, "Command \"%s\" exited with an error code of %u",
                newCmdline, exitCode);

            /* remove the temporary file, in case it was partially created */
            remove(tempFilename);

            return MARKUP_ERROR;
        }
    }

    /* include the output file like it was anything else */
    /* first, open it */
    if(OpenFile(tempFilename, tempFilename, "r", &infile) == FALSE)
    {
        HtpMsg(MSG_ERROR, task->infile, "unable to open execute result file");
        return MARKUP_ERROR;
    }

    /* build a new task */
    newTask.infile = &infile;
    newTask.outfile = task->outfile;
    newTask.varstore = task->varstore;
    newTask.sourceFilename = task->sourceFilename;

    /* process the file */
    result = ProcessTask(&newTask);

    /* close and destroy the output file */
    CloseFile(&infile);
    remove(tempFilename);

    return (result == TRUE) ? DISCARD_MARKUP : MARKUP_ERROR;
}

uint FileTemplateProcessor(TASK *task, HTML_MARKUP *htmlMarkup)
{
    const char *templateFile;

    assert(task != NULL);
    assert(htmlMarkup != NULL);

    if((templateFile = MarkupAttributeValue(htmlMarkup, "TEMPLATE")) == NULL)
    {
        HtpMsg(MSG_ERROR, task->infile, "a template file must be specified");
        return MARKUP_ERROR;
    }

    /* the template file is not actually processed now, but rather when the */
    /* rest of the file is completed ... to postpone processing, the template */
    /* name is kept in the variable store under a special name and retrieved */
    /* later */
    if(StoreVariable(task->varstore, VAR_TEMPLATE_NAME, templateFile,
        VAR_TYPE_INTERNAL, 0, NULL, NULL) == FALSE)
    {
        HtpMsg(MSG_ERROR, task->infile,
            "unable to store template filename for post-processing (out of memory?)");
        return MARKUP_ERROR;
    }

    return DISCARD_MARKUP;
}

uint FileIncludeProcessor(TASK *task, HTML_MARKUP *htmlMarkup)
{
    TEXTFILE incfile;
    BOOL result;
    TASK newTask;
    char fullPathname[MAX_PATHNAME_LEN];
    const char *attribValue;
    VARSTORE varstore;
    VARSTORE *topVarstore;
    uint ctr;
    uint flag;

    /* get the filename to include */
    attribValue = MarkupAttributeValue(htmlMarkup, "INCLUDE");
    if(attribValue == NULL)
    {
        HtpMsg(MSG_ERROR, task->infile, "include filename not specified");
        return MARKUP_ERROR;
    }

    /* open the include file as input */
    if(OpenFile(attribValue, attribValue, "r", &incfile) == FALSE)
    {
        /* use the search path to find the file */
        if(SearchForFile(attribValue, fullPathname, MAX_PATHNAME_LEN) == FALSE)
        {
            /* could not find the file in the search path either */
            HtpMsg(MSG_ERROR, task->infile, "unable to open include file \"%s\"",
                attribValue);
            return MARKUP_ERROR;
        }

        if(OpenFile(fullPathname, fullPathname, "r", &incfile) == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to open include file \"%s\"",
                fullPathname);
            return MARKUP_ERROR;
        }
    }
    else
    {
        StringCopy(fullPathname, attribValue, MAX_PATHNAME_LEN);
    }

    /* if additional parameters exist in the tag, build a local varstore */
    /* and push it onto the context */
    if(htmlMarkup->attribCount > 1)
    {
        if(InitializeVariableStore(&varstore) == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to initialize context for include file");
            CloseFile(&incfile);
            return MARKUP_ERROR;
        }

        /* start including the parameters as local macros */
        for(ctr = 1; ctr < htmlMarkup->attribCount; ctr++)
        {
            flag = (htmlMarkup->attrib[ctr].quoted == TRUE) ? VAR_FLAG_QUOTED
                : VAR_FLAG_NONE;
            if(StoreVariable(&varstore, htmlMarkup->attrib[ctr].name,
                htmlMarkup->attrib[ctr].value, VAR_TYPE_SET_MACRO,
                flag, NULL, NULL) == FALSE)
            {
                HtpMsg(MSG_ERROR, task->infile, "unable to add variable to context for include file");
                CloseFile(&incfile);
                DestroyVariableStore(&varstore);
                return MARKUP_ERROR;
            }
        }

        /* push this onto the context and use it for the include file */
        PushVariableStoreContext(task->varstore, &varstore);
        topVarstore = &varstore;
    }
    else
    {
        topVarstore = task->varstore;
    }

    /* build a new task structure */
    newTask.infile = &incfile;
    newTask.outfile = task->outfile;
    newTask.varstore = topVarstore;
    newTask.sourceFilename = task->sourceFilename;

    /* informational message for the user */
    HtpMsg(MSG_INFO, task->infile, "including file \"%s\"", fullPathname);

    /* process the new input file */
    result = ProcessTask(&newTask);

    /* pop the local context */
    if(topVarstore == &varstore)
    {
        assert(PeekVariableStoreContext(topVarstore) == topVarstore);
        PopVariableStoreContext(topVarstore);
        DestroyVariableStore(&varstore);
    }

    CloseFile(&incfile);

    /* if the new file did not process, return an error, otherwise discard */
    /* the markup */
    return (result == TRUE) ? DISCARD_MARKUP : MARKUP_ERROR;
}

uint FileProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    const char *attribName;
    const char *attribValue;
    const char *externalName;
    struct tm *timeNow;
    struct stat fileStat;

    /* if a NAME attribute is found, and it contains a value, use the */
    /* ExternalFileProcessor to create the plaintext (this function only */
    /* reports output file's time, date, name) */
    if(IsAttributeInMarkup(htmlMarkup, "NAME"))
    {
        if((externalName = MarkupAttributeValue(htmlMarkup, "NAME")) != NULL)
        {
            return ExternalFileProcessor(task, htmlMarkup, externalName,
                newPlaintext);
        }
    }

    /* if NAME attribute not in markup, or no external filename specified, */
    /* only one attribute can be used: NAME, SIZE, TIME, DATE , or INCLUDE */
    /* (the exception being EXECUTE and TEMPLATE) */
    if(IsAttributeInMarkup(htmlMarkup, "EXECUTE") == TRUE)
    {
        return FileExecuteProcessor(task, htmlMarkup);
    }

    if(IsAttributeInMarkup(htmlMarkup, "TEMPLATE") == TRUE)
    {
        return FileTemplateProcessor(task, htmlMarkup);
    }

    if(IsAttributeInMarkup(htmlMarkup, "INCLUDE") == TRUE)
    {
        return FileIncludeProcessor(task, htmlMarkup);
    }

    if(htmlMarkup->attribCount != 1)
    {
        HtpMsg(MSG_ERROR, task->infile, "improper FILE syntax");
        return MARKUP_ERROR;
    }

    /* get the attribute */
    attribName = htmlMarkup->attrib[0].name;
    attribValue = htmlMarkup->attrib[0].value;

    /* act on the attribute */
    if(stricmp(attribName, "SEARCH") == 0)
    {
        /* set the include search path to what was specified */
        if(attribValue != NULL)
        {
            StringCopy(searchPath, attribValue, SEARCH_PATH_SIZE);
        }
        else
        {
            /* search path is cleared */
            searchPath[0] = NUL;
        }

        return DISCARD_MARKUP;
    }
    else
    {
        /* NAME, TIME, DATE or bad tag */
        /* first, allocate some space for the (possibly) new markup */
        if((*newPlaintext = AllocMemory(MAX_TIME_DATE_SIZE)) == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to allocate memory for expansion");
            return MARKUP_ERROR;
        }

        /* get the input files time, in case the attribute is TIME or DATE */
        if(stat(task->sourceFilename, &fileStat) != 0)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to get information for file \"%s\"\n",
                task->infile->filename);
            return MARKUP_ERROR;
        }

        timeNow = localtime(&fileStat.st_mtime);

        if(stricmp(attribName, "TIME") == 0)
        {
            if(attribValue != NULL)
            {
                strftime(*newPlaintext, MAX_TIME_DATE_SIZE, attribValue, timeNow);
            }
            else
            {
                strftime(*newPlaintext, MAX_TIME_DATE_SIZE, "%I:%M:%S %p", timeNow);
            }

            HtpMsg(MSG_INFO, task->outfile, "adding local time");
        }
        else if(stricmp(attribName, "DATE") == 0)
        {
            if(attribValue != NULL)
            {
                strftime(*newPlaintext, MAX_TIME_DATE_SIZE, attribValue, timeNow);
            }
            else
            {
                strftime(*newPlaintext, MAX_TIME_DATE_SIZE, "%a %b %d, %Y", timeNow);
            }

            HtpMsg(MSG_INFO, task->outfile, "adding local date");
        }
        else if(stricmp(attribName, "NAME") == 0)
        {
            StringCopy(*newPlaintext, task->outfile->name, MAX_TIME_DATE_SIZE);

            HtpMsg(MSG_INFO, task->outfile, "adding output filename");
        }
        else
        {
            /* no appropriate tags found */
            HtpMsg(MSG_ERROR, task->infile, "invalid FILE tag attribute \"%s\"",
                attribName);

            /* free the allocated plaintext buffer */
            FreeMemory(*newPlaintext);
            *newPlaintext = NULL;

            return MARKUP_ERROR;
        }
    }

    /* the new plaintext has been created */
    return NEW_MARKUP;
}   

uint SetProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    char *name;
    char *value;
    uint ctr;
    uint flag;
    HTML_ATTRIBUTE *attrib;

    UNREF_PARAM(newPlaintext);

    /* have to declare at least one macro, but more are acceptable */
    if(htmlMarkup->attribCount == 0)
    {
        HtpMsg(MSG_ERROR, task->infile, "incomplete macro declaration");
        return MARKUP_ERROR;
    }

    attrib = &htmlMarkup->attrib[0];

    /* walk the list and add each macro to the variable store */
    for(ctr = 0; ctr < htmlMarkup->attribCount; ctr++)
    {
        /* get a private copy of the macro name */
        if((name = DuplicateString(attrib->name)) == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to store macro's value (out of memory?)");
            return MARKUP_ERROR;
        }

        value = attrib->value;
        flag = (attrib->quoted) ? VAR_FLAG_QUOTED : VAR_FLAG_NONE;

        /* put the new variable into the store */
        if(StoreVariable(task->varstore, name, value, VAR_TYPE_SET_MACRO, flag,
            NULL, NULL) == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to store macro \"%s\" (out of memory?)",
                name);
            FreeMemory(name);
            return MARKUP_ERROR;
        }

        if(value != NULL)
        {
            HtpMsg(MSG_INFO, task->infile, "macro \"%s\" assigned value \"%s\"",
                name, value);
        }
        else
        {
            HtpMsg(MSG_INFO, task->infile, "macro \"%s\" created with null value",
                name);
        }

        FreeMemory(name);

        attrib++;
    }

    return DISCARD_MARKUP;
}   

uint UnsetProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    uint ctr;
    uint type;
    const char *name;

    UNREF_PARAM(newPlaintext);

    /* have to specify at least one macro to remove, but more are acceptable */
    if(htmlMarkup->attribCount == 0)
    {
        HtpMsg(MSG_ERROR, task->infile, "UNSET tag improperly specified");
    }

    /* walk the attributes list */
    for(ctr = 0; ctr < htmlMarkup->attribCount; ctr++)
    {
        name = htmlMarkup->attrib[ctr].name;

        /* verify that the variable exists */
        if(VariableExists(task->varstore, name) == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile, "bad macro name \"%s\"", name);
            return MARKUP_ERROR;
        }

        /* remove it from the variable store if its not a DEF macro */
        type = GetVariableType(task->varstore, name);

        if((type == VAR_TYPE_SET_MACRO) || (type == VAR_TYPE_BLOCK_MACRO))
        {
            RemoveVariable(task->varstore, name);
        }

        HtpMsg(MSG_INFO, task->infile, "variable \"%s\" removed", name);
    }

    return DISCARD_MARKUP;
}

uint UseProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    char *name;
    const char *value;
    TEXTFILE incfile;
    int result;
    uint type;
    VARSTORE varstore;
    VARSTORE *topVarstore;
    uint ctr;
    uint flag;

    /* must declare at least 1 attribute, the macro name */
    if(htmlMarkup->attribCount == 0)
    {
        HtpMsg(MSG_ERROR, task->infile, "macro declaration not complete");
        return MARKUP_ERROR;
    }

    /* a variable reference should NOT include a new declaration */
    if(htmlMarkup->attrib[0].value != NULL)
    {
        HtpMsg(MSG_ERROR, task->infile, "improper USE syntax");
        return MARKUP_ERROR;
    }

    /* get a private copy of the name variable, this function will modify */
    /* its own copy */
    if((name = DuplicateString(htmlMarkup->attrib[0].name)) == NULL)
    {
        HtpMsg(MSG_ERROR, task->infile, "unable to expand macro (out of memory?)");
        return MARKUP_ERROR;
    }

    /* verify the macro exists */
    if(VariableExists(task->varstore, name) == FALSE)
    {
        HtpMsg(MSG_ERROR, task->infile, "macro %s has not been declared", name);
        FreeMemory(name);
        return MARKUP_ERROR;
    }

    /* get the value of the macro */
    value = GetVariableValue(task->varstore, name);

    /* get the type of macro */
    type = GetVariableType(task->varstore, name);

    if(type == VAR_TYPE_INTERNAL)
    {
        /* oof ... the user picked a variable name we use internally */
        /* !! a fix is to use both type and name as a key to get variable */
        /* out of hash, and therefore the name is not the only identifier */
        /* this will have to wait for later */
        HtpMsg(MSG_ERROR, task->infile,
            "reserved variable name \"%s\" used ... please use different name in HTP file",
            name);
        FreeMemory(name);
        return MARKUP_ERROR;
    }

    if(type == VAR_TYPE_DEF_MACRO)
    {
        /* nope */
        HtpMsg(MSG_ERROR, task->infile,
            "illegal to dereference a DEF macro with USE");
        FreeMemory(name);
        return MARKUP_ERROR;
    }

    assert((type == VAR_TYPE_SET_MACRO) || (type == VAR_TYPE_BLOCK_MACRO));

    /* if more than one parameter is on the USE tag, then assume they are */
    /* local variables for the macro */
    if(htmlMarkup->attribCount > 1)
    {
        if(type == VAR_TYPE_SET_MACRO)
        {
            /* nope, not yet */
            HtpMsg(MSG_ERROR, task->infile, "macro parameters can only be used for BLOCK macros");
            FreeMemory(name);
            return MARKUP_ERROR;
        }

        /* create a local variable store */
        if(InitializeVariableStore(&varstore) == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to initialize local context for macro");
            FreeMemory(name);
            return MARKUP_ERROR;
        }

        /* add each additional parameter to the local varstore */
        for(ctr = 1; ctr < htmlMarkup->attribCount; ctr++)
        {
            flag = (htmlMarkup->attrib[ctr].quoted == TRUE) ? VAR_FLAG_QUOTED
                : VAR_FLAG_NONE;
            if(StoreVariable(&varstore, htmlMarkup->attrib[ctr].name,
                htmlMarkup->attrib[ctr].value, VAR_TYPE_SET_MACRO, flag,
                NULL, NULL) == FALSE)
            {
                HtpMsg(MSG_ERROR, task->infile, "unable to add variable to block's local context");
                DestroyVariableStore(&varstore);
                FreeMemory(name);
                return MARKUP_ERROR;
            }
        }

        /* make this variable store the topmost context */
        PushVariableStoreContext(task->varstore, &varstore);
        topVarstore = &varstore;
    }
    else
    {
        topVarstore = task->varstore;
    }

    if(type == VAR_TYPE_SET_MACRO)
    {
        /* if NULL, then the macro was declared with no value, this is okay, */
        /* just don't do anything */
        if(value == NULL)
        {
            FreeMemory(name);
            return DISCARD_MARKUP;
        }

        /* allocate the new plaintext buffer and copy in the macro value */
        if((*newPlaintext = DuplicateString(value)) == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to allocate memory for macro expansion");
            FreeMemory(name);
            return MARKUP_ERROR;
        }

        HtpMsg(MSG_INFO, task->infile, "macro \"%s\" dereferenced", name);

        FreeMemory(name);

        return NEW_MARKUP;
    }
    else if(type == VAR_TYPE_BLOCK_MACRO)
    {
        /* !! magic number */
        char blockName[128];
        TASK newTask;

        /* if NULL, big-time error */
        if(value == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "block macro \"%s\" incorrectly stored, fatal internal error",
                name);
            if(topVarstore == &varstore)
            {
                assert(PeekVariableStoreContext(topVarstore) == topVarstore);
                PopVariableStoreContext(topVarstore);
                DestroyVariableStore(&varstore);
            }
            FreeMemory(name);
            exit(1);
        }

        /* build the block macro name (printed in place of the temporary */
        /* filename in all messages regarding it) */
        sprintf(blockName, "Block macro \"%s\"", name);

        /* block macro value is the name of a temporary file containing */
        /* macro contents */
        if(OpenFile(blockName, value, "r", &incfile) == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile,
                "unable to open temporary file \"%s\" for block macro \"%s\"",
                value, name);
            FreeMemory(name);
            return MARKUP_ERROR;
        }

        HtpMsg(MSG_INFO, task->infile, "dereferencing block macro \"%s\"", name);

        /* build a new task structure */
        newTask.infile = &incfile;
        newTask.outfile = task->outfile;
        newTask.sourceFilename = task->sourceFilename;

        /* re-use current variable store if no local variable store was */
        /* allocated, otherwise use the new one */
        newTask.varstore = topVarstore;

        /* process the new input file */
        result = ProcessTask(&newTask);

        /* remove the new context (and make sure it is, in fact, the block's */
        /* context) */
        if(topVarstore == &varstore)
        {
            assert(PeekVariableStoreContext(topVarstore) == topVarstore);
            PopVariableStoreContext(topVarstore);
            DestroyVariableStore(&varstore);
        }

        CloseFile(&incfile);

        FreeMemory(name);

        /* if the new file did not process, return an error, otherwise discard */
        /* the markup */
        return (result == TRUE) ? DISCARD_MARKUP : MARKUP_ERROR;
    }
    else
    {
        /* fatal error */
        printf("%s: fatal internal error (USE)\n", PROGRAM_NAME);
        FreeMemory(name);
        exit(1);

        /* to prevent compiler warning */
        return MARKUP_ERROR;
    }
}   

/*
// Block macro destructor callback ... used whenever a block macro is
// destroyed with a RemoveVariable() or ClearVariableList()
*/
void BlockDestructor(const char *name, const char *value, uint type, uint flags,
    void *param)
{
    UNREF_PARAM(name);
    UNREF_PARAM(type);
    UNREF_PARAM(flags);

    /* in debug versions of htp, a special "secret" option can be set to */
    /* keep BLOCK temporary files around, for later inspection ... */
    /* not real useful in release versions */
#if DEBUG
    if(KEEPTEMP == FALSE)
    {
        remove(value);
    }
    else
    {
        HtpMsg(MSG_INFO, NULL, "DEBUG: keeping temporary file %s", value);
    }
#else
    /* simply delete the temporary file holding the block macro text */
    DEBUG_PRINT(("deleting file %s for block macro %s\n", value, name));
    remove(value);
#endif

    /* DEF macros use param */
    if(param != NULL)
    {
        FreeMemory(param);
    }
}

uint BlockProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    char *name;
    char *plaintext;
    char newfile[MAX_PATHNAME_LEN];
    HTML_MARKUP newHtml;
    TEXTFILE blockFile;
    BOOL result;
    static uint blockDepth = 0;
    uint markupType;
    BOOL blockMacro;
    uint macroType;
    const char *macroTypeName;
    const char *openTag;
    const char *closeTag;
    const char *defName;
    char *param;

    UNREF_PARAM(newPlaintext);

    /* first: is this a BLOCK macro or a DEF macro?  This function is */
    /* overloaded to handle both types, and must internally change its */
    /* functionality for each type */
    /* if this is false, this is a DEF macro */
    if(stricmp(htmlMarkup->tag, "BLOCK") == 0)
    {
        blockMacro = TRUE;
        macroType = VAR_TYPE_BLOCK_MACRO;
        macroTypeName = "BLOCK";
        openTag = "BLOCK";
        closeTag = "/BLOCK";
    }
    else
    {
        assert(stricmp(htmlMarkup->tag, "DEF") == 0);
        blockMacro = FALSE;
        macroType = VAR_TYPE_DEF_MACRO;
        macroTypeName = "DEF";
        openTag = "DEF";
        closeTag = "/DEF";
    }

    /* check markup */
    if(blockMacro == TRUE)
    {
        /* is a name specified? (only one name can be specified) */
        if(htmlMarkup->attribCount != 1)
        {
            HtpMsg(MSG_ERROR, task->infile, "bad BLOCK markup specified");
            return MARKUP_ERROR;
        }

        /* no extra varstore parameter for a block macro */
        param = NULL;
    }
    else
    {
        /* DEF requires at least one parameter */
        if(htmlMarkup->attribCount == 0)
        {
            HtpMsg(MSG_ERROR, task->infile, "bad DEF markup specified");
            return MARKUP_ERROR;
        }

        /* check that the NAME attribute is present */
        if(IsAttributeInMarkup(htmlMarkup, "NAME") == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile, "DEF requires NAME attribute");
            return MARKUP_ERROR;
        }

        if(MarkupAttributeValue(htmlMarkup, "NAME") == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "DEF requires a NAME attribute to have a value");
            return MARKUP_ERROR;
        }

        /* if OPTION is specified, squirrel it away with the macro */
        if(IsAttributeInMarkup(htmlMarkup, "OPTION") == TRUE)
        {
            if((param = DuplicateString(MarkupAttributeValue(htmlMarkup, "OPTION")))
                == NULL)
            {
                HtpMsg(MSG_ERROR, task->infile, "unable to create OPTION copy for DEF macro");
                return MARKUP_ERROR;
            }
        }
        else
        {
            param = NULL;
        }
    }

    /* create a temporary filename to save the text block into */
    if(CreateTempFilename(newfile, MAX_PATHNAME_LEN) == FALSE)
    {
        HtpMsg(MSG_ERROR, task->infile, "unable to generate temporary filename for %s macro",
            macroTypeName);
        FreeMemory(param);
        return MARKUP_ERROR;
    }

    /* try and create the temporary file */
    if(OpenFile(newfile, newfile, "w", &blockFile) == FALSE)
    {
        HtpMsg(MSG_ERROR, task->infile,
            "unable to create temporary file \"%s\" for %s macro", newfile,
            macroTypeName);
        FreeMemory(param);
        return MARKUP_ERROR;
    }

    /* need a private copy of the macro name */
    if(blockMacro == TRUE)
    {
        if((name = DuplicateString(htmlMarkup->attrib[0].name)) == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to allocate memory for %s macro",
                macroTypeName);
            FreeMemory(param);
            CloseFile(&blockFile);
            return MARKUP_ERROR;
        }
    }
    else
    {
        defName = MarkupAttributeValue(htmlMarkup, "NAME");
        assert(defName != NULL);

        if((name = DuplicateString(defName)) == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "unable to allocate memory for %s macro",
                macroTypeName);
            FreeMemory(param);
            CloseFile(&blockFile);
            return MARKUP_ERROR;
        }
    }

    /* store the block file name and the block macro name as a variable */
    if(StoreVariable(task->varstore, name, newfile, macroType, VAR_FLAG_NONE,
        param, BlockDestructor) == FALSE)
    {
        HtpMsg(MSG_ERROR, task->infile, "unable to store macro information (out of memory?)");
        FreeMemory(name);
        FreeMemory(param);
        CloseFile(&blockFile);
        return MARKUP_ERROR;
    }

    /* the name is no longer used, dont free param it will be freed in */
    /* the block destructor */
    FreeMemory(name);
    name = NULL;

    /* start copying the file into the temporary file, looking for the */
    /* BLOCK or /BLOCK tag if block macro, DEF or /DEF otherwise ... */
    /* just squirrel away all other tags and text */
    for(;;)
    {
        result = ReadHtmlFile(task->infile, &blockFile, &plaintext, &markupType);
        if(result == ERROR)
        {
            CloseFile(&blockFile);
            return MARKUP_ERROR;
        }
        else if(result == FALSE)
        {
            /* end-of-file encountered before end-of-block */
            HtpMsg(MSG_ERROR, task->infile, "EOF encountered before %s macro declaration closed",
                macroTypeName);
            CloseFile(&blockFile);
            return MARKUP_ERROR;
        }

        /* turn the plain text into HTML structure, but don't destroy */
        /* the plaintext, this will be used to copy into output file if */
        /* necessary */
        if(PlaintextToMarkup(plaintext, &newHtml) == FALSE)
        {
            /* memory alloc failed, most likely */
            HtpMsg(MSG_ERROR, task->infile, "could not process markup (out of memory?)");
            CloseFile(&blockFile);
            FreeMemory(plaintext);
            return MARKUP_ERROR;
        }

        if(markupType & MARKUP_TYPE_HTP)
        {
            /* check for embedded block declarations */
            if(IsMarkupTag(&newHtml, openTag))
            {
                /* add to the block macro depth and continue */
                blockDepth++;
            }
            else if(IsMarkupTag(&newHtml, closeTag) == TRUE)
            {
                if(blockDepth > 0)
                {
                    /* depth has decreased one */
                    blockDepth--;
                }
                else
                {
                    /* found the end of the macro block */
                    DestroyMarkupStruct(&newHtml);
                    FreeMemory(plaintext);
                    break;
                }
            }
        }

        /* if continuing, then the plaintext is put into the output stream */
        /* as-is ... there is no case where the processor continues scanning */
        /* but discards a markup */
        PutFileString(&blockFile, "%c%s%c", MARKUP_OPEN_DELIM(markupType),
            plaintext, MARKUP_CLOSE_DELIM(markupType));

        /* destroy the HTML markup, not needed any longer */
        DestroyMarkupStruct(&newHtml);

        /* destroy the plaintext buffer */
        FreeMemory(plaintext);
        plaintext = NULL;
    }

    CloseFile(&blockFile);

    return DISCARD_MARKUP;
}   

BOOL DiscardConditionalBlock(TEXTFILE *infile)
{
    char *plaintext;
    HTML_MARKUP htmlMarkup;
    BOOL result;
    uint embeddedConditionals;
    uint markupType;

    /* discard the block, looking for the matching ELSE or /IF statement */
    embeddedConditionals = 0;
    for(;;)
    {
        result = ReadHtmlFile(infile, NULL, &plaintext, &markupType);
        if(result == ERROR)
        {
            return FALSE;
        }
        else if(result == FALSE)
        {
            /* end-of-file before end-of-conditional ... error */
            HtpMsg(MSG_ERROR, infile, "EOF encountered before conditional closed");
            return FALSE;
        }

        if(PlaintextToMarkup(plaintext, &htmlMarkup) == FALSE)
        {
            /* memory alloc error */
            HtpMsg(MSG_ERROR, infile, "could not parse markup tag (out of memory?)");
            FreeMemory(plaintext);
            return FALSE;
        }

        /* FreeMemory the plaintext buffer, not needed any longer */
        FreeMemory(plaintext);
        plaintext = NULL;

        /* another conditional started? */
        if(IsMarkupTag(&htmlMarkup, "IF"))
        {
            embeddedConditionals++;
        }
        else if(IsMarkupTag(&htmlMarkup, "/IF"))
        {
            /* end of the conditional? */
            if(embeddedConditionals == 0)
            {
                DestroyMarkupStruct(&htmlMarkup);
                break;
            }
            embeddedConditionals--;
        }
        else if(IsMarkupTag(&htmlMarkup, "ELSE"))
        {
            /* start of TRUE block? */
            if(embeddedConditionals == 0)
            {
                DestroyMarkupStruct(&htmlMarkup);
                break;
            }
        }

        /* destroy and continue */
        DestroyMarkupStruct(&htmlMarkup);
    }

    return TRUE;
}   

uint BooleanProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    static uint conditionalLevel = 0;
    const char *value;
    uint type;
    BOOL condTrue;
    HTML_ATTRIBUTE *attrib;
    BOOL notTagFound;

    UNREF_PARAM(newPlaintext);

    condTrue = FALSE;

    /* conditionalLevel keeps track of boolean depth */
    if(conditionalLevel == 0)
    {
        if((IsMarkupTag(htmlMarkup, "/IF")) || (IsMarkupTag(htmlMarkup, "ELSE")))
        {
            HtpMsg(MSG_ERROR, task->infile, "conditional block must start with IF tag");
            return MARKUP_ERROR;
        }
    }

    if(IsMarkupTag(htmlMarkup, "IF"))
    {
        conditionalLevel++;

        /* this is an ugly way to handle the IF-IF NOT test, but will need */
        /* be cleaned up in the future */

        /* should either be one or two attributes in markup */
        if(htmlMarkup->attribCount == 0)
        {
            HtpMsg(MSG_ERROR, task->infile, "no conditional to test");
            return MARKUP_ERROR;
        }

        if(htmlMarkup->attribCount > 2)
        {
            HtpMsg(MSG_ERROR, task->infile, "too many items in conditional expression");
            return MARKUP_ERROR;
        }

        /* find the attribute to evaluate and search for NOT attribute */
        notTagFound = FALSE;
        attrib = NULL;
        if(stricmp(htmlMarkup->attrib[0].name, "NOT") == 0)
        {
            /* check to make sure the second attribute is present */
            if(htmlMarkup->attribCount == 1)
            {
                HtpMsg(MSG_ERROR, task->infile, "NOT listed, no conditional to test");
                return MARKUP_ERROR;
            }

            notTagFound = TRUE;
            attrib = &htmlMarkup->attrib[1];
        }
        else if(htmlMarkup->attribCount == 2)
        {
            if(stricmp(htmlMarkup->attrib[1].name, "NOT") == 0)
            {
                notTagFound = TRUE;
                attrib = &htmlMarkup->attrib[0];
            }
            else
            {
                /* this should have been the NOT expression */
                HtpMsg(MSG_ERROR, task->infile, "too many conditionals to test");
                return MARKUP_ERROR;
            }
        }
        else
        {
            attrib = &htmlMarkup->attrib[0];
        }

        /* get the macros associated value (NULL if macro not defined) */
        if((value = GetVariableValue(task->varstore, attrib->name)) != NULL)
        {
            type = GetVariableType(task->varstore, attrib->name);
        }
        else
        {
            type = VAR_TYPE_SET_MACRO;
        }

        /* if only a name is specified, only care if macro is defined */
        if(attrib->value == NULL)
        {
            condTrue = (value != NULL) ? TRUE : FALSE;
        }
        else
        {
            /* macro value comparison */
            if(type == VAR_TYPE_SET_MACRO)
            {
                /* macro comparison (case-sensitive) */
                condTrue = (strcmp(value, attrib->value) == 0) ? TRUE : FALSE;
            }
            else
            {
                /* block macro, comparisons not allowed */
                condTrue = FALSE;
            }
        }

        /* reverse conditional if NOT attribute found */
        if(notTagFound == TRUE)
        {
            condTrue = (condTrue == TRUE) ? FALSE : TRUE;
        }

        if(condTrue == TRUE)
        {
            /* simply discard the markup and let the file processor continue */
            return DISCARD_MARKUP;
        }

        /* discard the rest of the conditional block since this portion has */
        /* evaluated false */
        if(DiscardConditionalBlock(task->infile) == FALSE)
        {
            return MARKUP_ERROR;
        }
    }
    else if(IsMarkupTag(htmlMarkup, "ELSE"))
    {
        /* this can only occur if the associated conditional statement */
        /* evaluated TRUE, so the remaining block must be discarded */
        if(DiscardConditionalBlock(task->infile) == FALSE)
        {
            return MARKUP_ERROR;
        }
    }
    else
    {
        /* end of conditional */
        assert(conditionalLevel > 0);
        conditionalLevel--;
    }

    return DISCARD_MARKUP;
}   

uint CommentProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    UNREF_PARAM(htmlMarkup);
    UNREF_PARAM(newPlaintext);

    /* authors ego-gratifying easter egg */
    /* put a final comment at the end of the output file */
    PutFileString(task->outfile, "\n\n<!-- HTML pre-processed by %s %d.%02d %s -->\n\n",
        PROGRAM_NAME, VER_MAJOR, VER_MINOR, VER_STAGE);

    return MARKUP_OKAY;
}

uint ConditionalWarning(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    UNREF_PARAM(htmlMarkup);
    UNREF_PARAM(newPlaintext);

    HtpMsg(MSG_ERROR, task->infile, "IFNOT tag no longer recognized; use IF NOT instead");
    return MARKUP_ERROR;
}

uint PreProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    UNREF_PARAM(htmlMarkup);
    UNREF_PARAM(newPlaintext);

    /* the CONDENSE option cannot be utilized inside a <PRE>...</PRE> */
    /* block, because there HTML DOES interpret CR's */
    if(CONDENSE)
    {
        if(IsMarkupTag(htmlMarkup, "PRE"))
        {
            AllowLinefeeds(task->outfile);
        }
        else if(IsMarkupTag(htmlMarkup, "/PRE"))
        {
            SuppressLinefeeds(task->outfile);
        }
    }

    return MARKUP_OKAY;
}

uint AltTextProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    const char *imgName;
    const char *imgText;

    UNREF_PARAM(newPlaintext);

    /* requires at least a NAME parameter */
    if(IsAttributeInMarkup(htmlMarkup, "NAME") == FALSE)
    {
        HtpMsg(MSG_ERROR, task->infile, "ALTTEXT requires a NAME attribute");
        return MARKUP_ERROR;
    }

    /* get the relevant information */
    imgName = MarkupAttributeValue(htmlMarkup, "NAME");
    if(imgName == NULL)
    {
        HtpMsg(MSG_ERROR, task->infile, "NAME must be specified with a filename");
        return MARKUP_ERROR;
    }

    if(IsAttributeInMarkup(htmlMarkup, "TEXT") == TRUE)
    {
        imgText = MarkupAttributeValue(htmlMarkup, "TEXT");
        if(imgText == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "TEXT must be specified with a value (\"\" is acceptable)");
            return MARKUP_ERROR;
        }
    }
    else
    {
        imgText = NULL;
    }

    /* try to find the graphic name in the ALTTEXT store */
    if(VariableExists(&altTextVarStore, imgName) == TRUE)
    {
        /* if no name specified, delete it from the store */
        if(imgText == NULL)
        {
            RemoveVariable(&altTextVarStore, imgName);
            HtpMsg(MSG_INFO, task->infile, "ALT text for image \"%s\" removed",
                imgName);
            return DISCARD_MARKUP;
        }

        /* since it exists, simply re-storing the value will delete the */
        /* old one and replace with new one */
    }
    else if(imgText == NULL)
    {
        /* tried to delete an image not already in the store */
        /* just post a warning */
        HtpMsg(MSG_WARNING, task->infile, "attempted to delete image text not already defined");
        return DISCARD_MARKUP;
    }

    StoreVariable(&altTextVarStore, imgName, imgText, VAR_TYPE_ALTTEXT,
        VAR_FLAG_NONE, NULL, NULL);

    HtpMsg(MSG_INFO, task->infile, "ALT text for image \"%s\" set to \"%s\"",
        imgName, imgText);

    return DISCARD_MARKUP;
}

uint UndefProcessor(TASK *task, HTML_MARKUP *htmlMarkup, char **newPlaintext)
{
    uint ctr;
    const char *name;

    UNREF_PARAM(newPlaintext);

    /* need at least one attribute to undef */
    if(htmlMarkup->attribCount == 0)
    {
        HtpMsg(MSG_ERROR, task->infile, "UNDEF requires at least one name");
        return MARKUP_ERROR;
    }

    /* walk the list of attributes, deleting them as found */
    for(ctr = 0; ctr < htmlMarkup->attribCount; ctr++)
    {
        name = htmlMarkup->attrib[ctr].name;

        /* is it in the store? */
        if(VariableExists(task->varstore, name) == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile, "No metatag \"%s\" to undefine",
                name);
            return MARKUP_ERROR;
        }

        /* only remove it if a DEF macro */
        if(GetVariableType(task->varstore, name) == VAR_TYPE_DEF_MACRO)
        {
            RemoveVariable(task->varstore, name);
        }

        HtpMsg(MSG_INFO, task->infile, "metatag \"%s\" removed", name);
    }

    return DISCARD_MARKUP;
}

#define MARKUP_PROCESSOR_COUNT          (17)
MARKUP_PROCESSORS markupProcessor[MARKUP_PROCESSOR_COUNT] =
{
    { "IMG", MARKUP_TYPE_HTML, ImageProcessor },
    { "OPT", MARKUP_TYPE_HTP, OptionProcessor },
    { "FILE", MARKUP_TYPE_HTP, FileProcessor },
    { "SET", MARKUP_TYPE_HTP, SetProcessor },
    { "BLOCK", MARKUP_TYPE_HTP, BlockProcessor },
    { "USE", MARKUP_TYPE_HTP, UseProcessor },
    { "IF", MARKUP_TYPE_HTP, BooleanProcessor },
    { "/IF", MARKUP_TYPE_HTP, BooleanProcessor },
    { "IFNOT", MARKUP_TYPE_HTP, ConditionalWarning },
    { "ELSE", MARKUP_TYPE_HTP, BooleanProcessor },
    { "/BODY", MARKUP_TYPE_HTML, CommentProcessor },
    { "UNSET", MARKUP_TYPE_HTP, UnsetProcessor },
    { "PRE", MARKUP_TYPE_HTML, PreProcessor },
    { "/PRE", MARKUP_TYPE_HTML, PreProcessor },
    { "ALTTEXT", MARKUP_TYPE_HTP, AltTextProcessor },
    { "DEF", MARKUP_TYPE_HTP, BlockProcessor },
    { "UNDEF", MARKUP_TYPE_HTP, UndefProcessor }
};

/*
// HTML processing
*/

BOOL ExpandMacrosInString(TASK *task, const char *text, char *newText,
    uint newTextSize, BOOL *quoted, uint *changeCount)
{
    const char *expansion;
    char macro[MAX_VARVALUE_LEN];
    char *textPtr;
    uint insertStartPos;
    uint insertEndPos;
    uint newTextLength;

    assert(task != NULL);
    assert(text != NULL);
    assert(newText != NULL);
    assert(newTextSize > 0);
    assert(quoted != NULL);
    assert(changeCount != NULL);

    *changeCount = 0;

    insertStartPos = 0;
    insertEndPos = 0;

    /* optimization: if no '$' in text, no need to go futher */
    if(strchr(text, '$') == NULL)
    {
#if DEBUG
        expandSkipped++;
#endif
        return TRUE;
    }

#if DEBUG
    expandPerformed++;
#endif

    /* Allan Todd fix: only check buffer size when its known to be needed */
    assert(newTextSize >= strlen(text));

    /* copy the old text directly into the new text buffer, and use that */
    /* as working space */
    StringCopy(newText, text, newTextSize);

    /* loop repeatedly to evaluate the string until no more macros are found */
    for(;;)
    {
        macro[0] = NUL;

        /* search the value for a $-preceded macro name */
        textPtr = strchr(newText, '$');
        while(textPtr != NULL)
        {
            /* if only one contiguous '$' found, stop looking and process */
            if(textPtr[1] != '$')
            {
                break;
            }

            textPtr = strchr(textPtr + 2, '$');
        }

        /* if nothing found, exit */
        if(textPtr == NULL)
        {
            break;
        }

        /* this is used later more than once, but could change each iteration */
        newTextLength = strlen(newText);

        /* process the macro */

        /* save the position to insert the macro */
        insertStartPos = textPtr - newText;

        /* skip the '$' */
        textPtr++;

        /* copy macro name into macro[] array and set up the text pointer */
        /* macro specified with braces? */
        if(*textPtr == '{')
        {
            char *endPtr;

            /* start of a macro */
            /* skip opening curly brace and find the closing curly brace */
            endPtr = strchr(++textPtr, '}');
            if(endPtr == NULL)
            {
                HtpMsg(MSG_ERROR, task->infile, "ending brace not found in macro name");
                return FALSE;
            }

            /* copy in the enclosed text */
            StringCopy(macro, textPtr, endPtr - textPtr + 1);

            /* set end insertion point */
            insertEndPos = endPtr - newText + 1;
        }
        else
        {
            /* start of macro, no braces */
            /* copy macro name until EOS */
            StringCopy(macro, textPtr, MAX_VARVALUE_LEN);

            /* set end insertion point */
            insertEndPos = newTextLength;
        }

        /* if no macro name found, stop */
        if(macro[0] == NUL)
        {
            break;
        }

        /* make sure variable exists in store */
        if(VariableExists(task->varstore, macro) != TRUE)
        {
            /* only a warning ... stop processing to prevent infinite */
            /* loop */
            HtpMsg(MSG_WARNING, task->infile, "unrecognized macro name \"%s\"",
                macro);
            break;
        }

        /* block macros cannot be expanded inside of markup tags */
        if(GetVariableType(task->varstore, macro) == VAR_TYPE_BLOCK_MACRO)
        {
            HtpMsg(MSG_ERROR, task->infile, "cannot expand block macro \"%s\" inside a markup",
                macro);
            return FALSE;
        }

        /* get the macros value and replace the attribute value */
        expansion = GetVariableValue(task->varstore, macro);

        if(expansion != NULL)
        {
            HtpMsg(MSG_INFO, task->infile, "expanding macro \"%s\" to \"%s\"",
                macro, expansion);
        }
        else
        {
            HtpMsg(MSG_INFO, task->infile, "expanding macro \"%s\" to null text",
                macro);
        }

        /* build a new string using the expanded macro */

        /* if the macro is embedded inside a larger string, */
        /* actually go through the rigamarole of piecing together */
        /* a new value string */
        if((insertStartPos != 0)
            || (insertEndPos != newTextLength))
        {
            char *dupString;

            /* make a copy of the text buffer */
            /* can't use DuplicateString() because extra buffer past EOS needed */
            dupString = AllocMemory(newTextSize);
            if(dupString == NULL)
            {
                HtpMsg(MSG_ERROR, task->infile, "out of memory during macro expansion");
                return FALSE;
            }
            StringCopy(dupString, newText, newTextSize);

            /* copy in the expanded macro at the insertion point */
            if(expansion != NULL)
            {
                StringCopy(dupString + insertStartPos, expansion,
                    newTextSize - insertStartPos);
            }
            else
            {
                dupString[insertStartPos] = NUL;
            }

            /* copy in any characters after the macro ended */
            strncat(dupString, newText + insertEndPos, newTextSize);

            /* copy it back to the old string buffer and free */
            StringCopy(newText, dupString, newTextSize);
            FreeMemory(dupString);
            dupString = NULL;

            /* let the surrounding text dictate quote marks */
        }
        else
        {
            /* since the macro is the entire value, no harm (and */
            /* more robust) to surround it by quotes */
            StringCopy(newText, expansion, MAX_VARVALUE_LEN);
            *quoted = TRUE;
        }

        /* increment the change count */
        *changeCount = *changeCount + 1;

        /* need to go back and re-evaluate the value for more macros */
    }

    /* guess what!  need to re-process the value again to strip the */
    /* double '$' ... this couldn't be done previously because the newly */
    /* single '$' would have been processed as macro as the loop went */
    /* back around to finish off the next dereferenced macro */
    textPtr = strchr(newText, '$');
    while(textPtr != NULL)
    {
        /* look for adjacent '$' */
        if(textPtr[1] == '$')
        {
            /* terminate the preceding characters */
            textPtr[1] = NUL;

            /* concatenate the rest of the string */
            memmove(textPtr + 1, textPtr + 2, strlen(textPtr + 2) + 1);

            /* increment the change count */
            *changeCount = *changeCount + 1;

            /* start looking at the beginning of the new string */
            textPtr = strchr(newText, '$');
            continue;
        }

        /* just keep looking otherwise */
        textPtr = strchr(textPtr + 1, '$');
    }

    return TRUE;
}

BOOL ExpandMacros(TASK *task, HTML_MARKUP *htmlMarkup)
{
    uint ctr;
    HTML_ATTRIBUTE *attrib;
    BOOL result;
    union
    {
        char newTag[MAX_VARNAME_LEN];
        char newName[MAX_VARNAME_LEN];
        char newValue[MAX_VARVALUE_LEN];
    } newText;
    BOOL quoted;
    BOOL changeCount;

    assert(task != NULL);
    assert(htmlMarkup != NULL);

    /* expand any macros in the tags */
    if(htmlMarkup->tag != NULL)
    {
        if(ExpandMacrosInString(task, htmlMarkup->tag, newText.newTag,
            MAX_VARNAME_LEN, &quoted, &changeCount) != TRUE)
        {
            return FALSE;
        }

        if(changeCount > 0)
        {
            ChangeMarkupTag(htmlMarkup, newText.newTag);
        }
    }

    /* do the same for all attributes, both name and value */
    result = TRUE;
    for(ctr = 0; ctr < htmlMarkup->attribCount; ctr++)
    {
        attrib = &htmlMarkup->attrib[ctr];

        if(attrib->name != NULL)
        {
            result = ExpandMacrosInString(task, attrib->name, newText.newName,
                MAX_VARNAME_LEN, &quoted, &changeCount);
            if(result != TRUE)
            {
                break;
            }

            if(changeCount > 0)
            {
                ChangeAttributeName(attrib, newText.newName);
            }
        }


        if(attrib->value != NULL)
        {
            quoted = attrib->quoted;
            result = ExpandMacrosInString(task, attrib->value, newText.newValue,
                MAX_VARVALUE_LEN, &quoted, &changeCount);
            if(result != TRUE)
            {
                break;
            }

            if(changeCount > 0)
            {
                ChangeAttributeValue(attrib, newText.newValue, quoted);
            }
        }
    } 

    return result;
}   

#define METATAG_MAX_OPTIONS                     (256)

uint ExpandMetatag(TASK *task, HTML_MARKUP *htmlMarkup)
{
    const char *options;
    char *optionCopy;
    FIND_TOKEN findToken;
    char *optionPtr;
    uint optionCount;
    uint ctr;
    uint optionCtr;
    const char *optionArray[METATAG_MAX_OPTIONS];
    const HTML_ATTRIBUTE *attrib;
    BOOL found;
    VARSTORE defVarstore;
    uint flag;
    VARSTORE *topVarstore;
    char defBlockName[64];
    const char *defFilename;
    const char *defName;
    TEXTFILE defFile;
    TASK newTask;
    BOOL result;

    /* first things first: find the tag in the metatag store */
    if(VariableExists(task->varstore, htmlMarkup->tag) == FALSE)
    {
        /* don't change a thing */
        return MARKUP_OKAY;
    }

    /* verify the macro in the store is a metatag definition */
    if(GetVariableType(task->varstore, htmlMarkup->tag) != VAR_TYPE_DEF_MACRO)
    {
        return MARKUP_OKAY;
    }

    /* get a pointer to the name */
    defName = htmlMarkup->tag;

    /* get the filename the DEF macro is held in */
    if((defFilename = GetVariableValue(task->varstore, defName)) == NULL)
    {
        /* this shouldnt be */
        HtpMsg(MSG_ERROR, task->infile, "DEF macro \"%s\" was not store properly",
            defName);
        return MARKUP_ERROR;
    }

    /* get options to compare against markups paramater list */
    options = GetVariableParam(task->varstore, defName);

    /* initialize a local variable store, even if its not used */
    InitializeVariableStore(&defVarstore);

    /* if NULL, then no options allowed */
    if(options == NULL)
    {
        if(htmlMarkup->attribCount > 0)
        {
            HtpMsg(MSG_ERROR, task->infile, "DEF macro \"%s\" does not specify any options",
                defName);
            DestroyVariableStore(&defVarstore);
            return MARKUP_ERROR;
        }

        /* keep the current store context */
        topVarstore = task->varstore;
    }
    else
    {
        /* options should be space-delimited, use StringToken() */
        if((optionCopy = DuplicateString(options)) == NULL)
        {
            HtpMsg(MSG_ERROR, task->infile, "Unable to duplicate option macro (out of memory?)");
            DestroyVariableStore(&defVarstore);
            return MARKUP_ERROR;
        }

        /* build array of pointers to null-terminated option */
        optionCount = 0;
        optionPtr = StringFirstToken(&findToken, optionCopy, " ");
        while((optionPtr != NULL) && (optionCount < METATAG_MAX_OPTIONS))
        {
            /* ignore multiple spaces */
            if(*optionPtr != ' ')
            {
                /* save a pointer to the token */
                optionArray[optionCount++] = optionPtr;
            }

            optionPtr = StringNextToken(&findToken);
        }

        /* now, see if every paramater in the markup is also in the option list */
        /* (the reverse is not required) */
        for(ctr = 0; (attrib = MarkupAttribute(htmlMarkup, ctr)) != NULL; ctr++)
        {
            found = FALSE;
            for(optionCtr = 0; optionCtr < optionCount; optionCtr++)
            {
                if(stricmp(optionArray[optionCtr], attrib->name) == 0)
                {
                    found = TRUE;
                    break;
                }
            }

            if(found == FALSE)
            {
                HtpMsg(MSG_ERROR, task->infile,
                    "DEF macro \"%s\" does not accept a parameter named \"%s\"",
                    defName, attrib->name);
                FreeMemory(optionCopy);
                DestroyVariableStore(&defVarstore);
                return MARKUP_ERROR;
            }

            /* since this is a good attribute, add it to the local store */
            flag = (attrib->quoted == TRUE) ? VAR_FLAG_QUOTED : VAR_FLAG_NONE;
            if(StoreVariable(&defVarstore, attrib->name, attrib->value,
                VAR_TYPE_SET_MACRO, flag, NULL, NULL) == FALSE)
            {
                HtpMsg(MSG_ERROR, task->infile,
                    "Unable to store local macro for metatag");
                DestroyVariableStore(&defVarstore);
                FreeMemory(optionCopy);
                return MARKUP_ERROR;
            }
        }

        /* looks good, this is no longer needed */
        FreeMemory(optionCopy);

        /* make this the topmost context */
        PushVariableStoreContext(task->varstore, &defVarstore);
        topVarstore = &defVarstore;
    }

    /* expand the DEF macro like a block macro ... */

    /* set up the DEF macro name for user messages */
    sprintf(defBlockName, "Metatag \"%s\"", defName);

    /* open the file the macro is held in */
    if(OpenFile(defBlockName, defFilename, "r", &defFile) == FALSE)
    {
        HtpMsg(MSG_ERROR, task->infile,
            "unable to open temporary file \"%s\" for DEF macro \"%s\"",
            defFilename, defName);
        return MARKUP_ERROR;
    }

    HtpMsg(MSG_INFO, task->infile, "dereferencing metatag macro \"%s\"", defName);

    /* build a new task structure */
    newTask.infile = &defFile;
    newTask.outfile = task->outfile;
    newTask.sourceFilename = task->sourceFilename;

    /* re-user current variable store if no local store was made */
    newTask.varstore = topVarstore;

    /* process the new input file */
    result = ProcessTask(&newTask);

    /* remove the new context if necessary */
    if(topVarstore == &defVarstore)
    {
        assert(PeekVariableStoreContext(topVarstore) == topVarstore);
        PopVariableStoreContext(topVarstore);
    }

    /* no matter what, destroy the local store */
    DestroyVariableStore(&defVarstore);

    CloseFile(&defFile);

    return (result == TRUE) ? DISCARD_MARKUP : MARKUP_ERROR;
}

BOOL ProcessTask(TASK *task)
{
    char *newPlaintext;
    uint ctr;
    uint markupResult;
    HTML_MARKUP htmlMarkup;
    BOOL result;
    uint markupType;

    assert(task != NULL);
    assert(task->infile != NULL);
    assert(task->outfile != NULL);
    assert(task->varstore != NULL);

    for(;;)
    {
        result = ReadHtmlFile(task->infile, task->outfile, &newPlaintext,
            &markupType);
        if(result == ERROR)
        {
            /* problem reading in the file */
            return FALSE;
        }
        else if(result == FALSE)
        {
            /* end-of-file */
            break;
        }

        /* use the new markup plain text to build an HTML_MARKUP structure */
        if(PlaintextToMarkup(newPlaintext, &htmlMarkup) == FALSE)
        {
            HtpMsg(MSG_ERROR, task->infile, "could not parse markup tag (out of memory?)");
            FreeMemory(newPlaintext);
            return FALSE;
        }

        /* destroy the ORIGINAL plain text, not needed again */
        FreeMemory(newPlaintext);
        newPlaintext = NULL;

        /* give the default processor a chance to expand macros, etc. */
        if(ExpandMacros(task, &htmlMarkup) == FALSE)
        {
            /* problem encountered trying to expand macros */
            DestroyMarkupStruct(&htmlMarkup);
            return FALSE;
        }

        /* give the metatag processor a chance to expand metatags */
        /* this is a little strange, but if MARKUP_OKAY it means the the */
        /* metatag processor didnt recognize the tag, and therefore should */
        /* be handled by the other processors */
        if((markupResult = ExpandMetatag(task, &htmlMarkup)) == MARKUP_OKAY)
        {
            /* find the first processor that wants to do something with the */
            /* markup tag */
            for(ctr = 0; ctr < MARKUP_PROCESSOR_COUNT; ctr++)
            {
                if(markupProcessor[ctr].markupType & markupType)
                {
                    if(IsMarkupTag(&htmlMarkup, markupProcessor[ctr].tag))
                    {
                        assert(markupProcessor[ctr].markupFunc != NULL);

                        markupResult = markupProcessor[ctr].markupFunc(task,
                            &htmlMarkup, &newPlaintext);

                        break;
                    }
                }
            }
        }

        /* unless the function requested to use its new markup string, */
        /* take the HTML_MARKUP structure and build a new markup */
        if((markupResult != NEW_MARKUP) && (markupResult != DISCARD_MARKUP))
        {
            if(MarkupToPlaintext(&htmlMarkup, &newPlaintext) == FALSE)
            {
                HtpMsg(MSG_ERROR, task->infile, "unable to build plain text from markup (out of memory?)");
                return FALSE;
            }
        }

        /* destroy the structure, now only interested in the markup string */
        DestroyMarkupStruct(&htmlMarkup);

        switch(markupResult)
        {
            case MARKUP_OKAY:
            {
                /* add the markup to the output file as it should appear */
                PutFileString(task->outfile, "%c%s%c", MARKUP_OPEN_DELIM(markupType),
                    newPlaintext, MARKUP_CLOSE_DELIM(markupType));
            }
            break;

            case NEW_MARKUP:
            case MARKUP_REPLACED:
            {
                /* the markup has been replaced by a normal string */
                PutFileString(task->outfile, "%s", newPlaintext);
            }
            break;

            case DISCARD_MARKUP:
            {
                /* markup will not be included in final output */
            }
            break;

            case MARKUP_ERROR:
            {
                /* (need to destroy plaintext buffer before exiting) */
                FreeMemory(newPlaintext);
                return FALSE;
            }

            default:
            {
                FreeMemory(newPlaintext);
                printf("%s: serious internal error\n", PROGRAM_NAME);
                exit(1);
            }
        }

        /* free the plain text buffer and continue with processing */
        FreeMemory(newPlaintext);
        newPlaintext = NULL;
    }

    return TRUE;
}

BOOL FullyCheckDependencies(const char *in, const char *out)
{
    BOOL result;
    TEXTFILE infile;
    char *plaintext;
    char title[128];
    HTML_MARKUP markup;
    const char *includeFile;
    const char *imageFile;
    BOOL readResult;
    BOOL checkResult;
    uint markupType;

    assert(in != NULL);
    assert(out != NULL);

    if(DEPEND == FALSE)
    {
        /* outta here */
        return FALSE;
    }

    /* check if target file is completely up to date compared to input file */
    result = IsTargetUpdated(in, out);
    if(result == ERROR)
    {
        printf("%s: unable to get file information for file \"%s\"\n",
            PROGRAM_NAME, in);
        return ERROR;
    }
    else if(result == FALSE)
    {
        /* target is not updated */
        return FALSE;
    }

    /* because target is up to date, need to search dependency file for */
    /* FILE INCLUDE tags and check those files likewise */

    /* open file */
    sprintf(title, "Dependency check for %s", in);
    if(OpenFile(title, in, "r", &infile) == FALSE)
    {
        printf("%s: unable to open file \"%s\" for reading while checking dependencies\n",
            PROGRAM_NAME, in);
        return ERROR;
    }

    /* assume everything is hunky-dory unless otherwise discovered */
    checkResult = TRUE;

    /* get the next markup tag from the input file */
    while((readResult = ReadHtmlFile(&infile, NULL, &plaintext, &markupType)) != FALSE)
    {
        if(readResult == ERROR)
        {
            /* error occurred processing the HTML file */
            checkResult = ERROR;
            break;
        }

        /* check markup type ... only interested in htp markups currently */
        if((markupType & MARKUP_TYPE_HTP) == 0)
        {
            continue;
        }

        /* received a markup ... check if its an INCLUDE markup */
        PlaintextToMarkup(plaintext, &markup);

        /* do not need plaintext any further */
        FreeMemory(plaintext);
        plaintext = NULL;

        /* if FILE INCLUDE markup, get the filename specified */
        includeFile = NULL;
        imageFile = NULL;
        if(IsMarkupTag(&markup, "FILE"))
        {
            if(IsAttributeInMarkup(&markup, "INCLUDE"))
            {
                includeFile = MarkupAttributeValue(&markup, "INCLUDE");
            }
            else if(IsAttributeInMarkup(&markup, "TEMPLATE"))
            {
                includeFile = MarkupAttributeValue(&markup, "TEMPLATE");
            }
        }
        else if(IsMarkupTag(&markup, "IMG"))
        {
            if(IsAttributeInMarkup(&markup, "SRC"))
            {
                imageFile = MarkupAttributeValue(&markup, "SRC");
            }
        }
        else if(IsMarkupTag(&markup, "OPT"))
        {
            if(IsAttributeInMarkup(&markup, "NODEPEND"))
            {
                /* !! dependency checking disabled in source file ... since this */
                /* can swing back and forth throughout the files, and its just */
                /* a pain to track what is technically the last one set, */
                /* if one is found, dependency checking is disabled and the */
                /* targets are not considered updated */
                /* this could be fixed with some work */
                checkResult = FALSE;
                DestroyMarkupStruct(&markup);
                break;
            }
        }

        /* by default assume everything is up to date unless more information */
        /* is available through other files */
        result = TRUE;

        /* check include or image file timestamps */
        if(includeFile != NULL)
        {
            /* !! the accursed recursion strikes again */
            /* check the dependencies based on this new file */
            result = FullyCheckDependencies(includeFile, out);
        }
        else if(imageFile != NULL)
        {
            /* check the image files timestamp as part of dependency checking */
            if(FileExists(imageFile))
            {
                result = IsTargetUpdated(imageFile, out);
            }
        }

        /* unneeded now */
        DestroyMarkupStruct(&markup);

        if(result != TRUE)
        {
            /* if FALSE, not up to date, no need to go further */
            /* if ERROR, need to stop and report to caller */
            checkResult = result;
            break;
        }

        /* otherwise, TRUE indicates that everything is okay, */
        /* so keep searching */
    }

    /* EOF encountered in the HTML input file ... target is updated */
    CloseFile(&infile);

    return checkResult;
}

BOOL ProcessFileByName(const char *in, const char *out)
{
    TEXTFILE infile;
    TEXTFILE outfile;
    TASK task;
    BOOL result;
    TEXTFILE project;
    const char *templateFile;

    assert(in != NULL);
    assert(out != NULL);

    /* before proceeding, find a local project default file */
    if(FileExists("htp.def"))
    {
        StringCopy(projectFilename, "htp.def", MAX_PATHNAME_LEN);
    }
    else
    {
        projectFilename[0] = NUL;
    }

    /* assume no processing required */
    result = TRUE;

    /* check the global and project default files first */
    if(*globalFilename != NUL)
    {
        if((result = FullyCheckDependencies(globalFilename, out)) == ERROR)
        {
            return FALSE;
        }
    }

    if((result == TRUE) && (*projectFilename != NUL))
    {
        if((result = FullyCheckDependencies(projectFilename, out)) == ERROR)
        {
            return FALSE;
        }
    }

    /* check the dependencies of the target file to see whether or not */
    /* to proceed ... the global and project default files are checked as well */
    if(result == TRUE)
    {
        if((result = FullyCheckDependencies(in, out)) == ERROR)
        {
            /* did not process the files */
            return FALSE;
        }
    }

    /* if TRUE, no need to go any further */
    if(result == TRUE)
    {
        /* explain why no processing required, and return as if processing */
        /* was completed */
        printf("%s: File \"%s\" is completely up to date.\n", PROGRAM_NAME,
            out);
        return TRUE;
    }

    /* continue, at least one file was found that requires out to be updated */

    /* initialize the project variable store and push it onto context */
    InitializeVariableStore(&projectVarStore);
    PushVariableStoreContext(&globalVarStore, &projectVarStore);

    /* initialize the ALT text store (used by the ALTTEXT tag) */
    InitializeVariableStore(&altTextVarStore);

    /* open the output file first, the project default file needs it */
    if(OpenFile(out, out, "w", &outfile) == FALSE)
    {
        printf("%s: unable to open file \"%s\" for writing\n", PROGRAM_NAME, out);
        DestroyVariableStore(&altTextVarStore);
        DestroyVariableStore(&projectVarStore);
        return FALSE;
    }

    if(CONDENSE)
    {
        /* suppress all linefeeds for this file, makes the HTML output smaller */
        SuppressLinefeeds(&outfile);
    }

    /* clear the task struct, in case there is no project file */
    memset(&task, 0, sizeof(TASK));

    if(projectFilename[0] != NUL)
    {
        /* process the default project file */
        if(OpenFile(projectFilename, projectFilename, "r", &project) == FALSE)
        {
            printf("%s: unable to open file \"%s\" for reading\n", PROGRAM_NAME,
                projectFilename);
            CloseFile(&outfile);
            DestroyVariableStore(&projectVarStore);
            DestroyVariableStore(&altTextVarStore);
            return FALSE;
        }

        /* build a task structure */
        task.infile = &project;
        task.outfile = &outfile;
        task.varstore = &projectVarStore;
        task.sourceFilename = in;

        printf("%s: Processing default project file \"%s\" ...\n", PROGRAM_NAME,
            projectFilename);

        result = ProcessTask(&task);

        CloseFile(&project);

        if(result != TRUE)
        {
            if(PRECIOUS == FALSE)
            {
                remove(out);
            }

            printf("%s: error during processing of default project file \"%s\"\n",
                PROGRAM_NAME, projectFilename);

            CloseFile(&outfile);
            DestroyVariableStore(&projectVarStore);
            DestroyVariableStore(&altTextVarStore);
            return result;
        }
    }

    if(OpenFile(in, in, "r", &infile) == FALSE)
    {
        printf("%s: unable to open file \"%s\" for reading\n", PROGRAM_NAME, in);
        CloseFile(&outfile);
        DestroyVariableStore(&projectVarStore);
        DestroyVariableStore(&altTextVarStore);
        return FALSE;
    }

    if(InitializeLocalOption() == FALSE)
    {
        printf("%s: unable to initialize local option store\n", PROGRAM_NAME);
        CloseFile(&infile);
        CloseFile(&outfile);
        DestroyVariableStore(&projectVarStore);
        DestroyVariableStore(&altTextVarStore);
        return FALSE;
    }

    /* build a task structure */
    task.infile = &infile;
    task.outfile = &outfile;
    task.varstore = &projectVarStore;
    task.sourceFilename = in;

    printf("%s: Processing file \"%s\" to output file \"%s\" ...\n",
        PROGRAM_NAME, in, out);

    result = ProcessTask(&task);

    /* need to check for a template file */
    while((result == TRUE) && (VariableExists(&projectVarStore, VAR_TEMPLATE_NAME)))
    {
        /* go process it */

        /* done with this file, want to reuse struct */
        CloseFile(&infile);

        templateFile = GetVariableValue(&projectVarStore, VAR_TEMPLATE_NAME);

        if(OpenFile(templateFile, templateFile, "r", &infile) == FALSE)
        {
            printf("%s: unable to open template file \"%s\"\n",
                PROGRAM_NAME, templateFile);

            CloseFile(&outfile);
            DestroyLocalOption();
            DestroyVariableStore(&altTextVarStore);
            DestroyVariableStore(&projectVarStore);

            return FALSE;
        }

        task.infile = &infile;
        task.outfile = &outfile;
        task.varstore = &projectVarStore;
        task.sourceFilename = in;

        printf("%s: Processing template file \"%s\" ...\n", PROGRAM_NAME,
            templateFile);

        result = ProcessTask(&task);

        /* because this template file can, legally, reference another */
        /* template file, remove the current variable and let the while loop */
        /* continue until no more template files are specified */
        /* yes, this can lead to infinite loops and such, but the syntax */
        /* shouldnt bar this, and after all, the same problem could exist if */
        /* the user kept doing circular FILE INCLUDEs */
        /* Pilot error! */
        RemoveVariable(&projectVarStore, VAR_TEMPLATE_NAME);
    }

    if(result == TRUE)
    {
        printf("%s: final output file \"%s\" successfully created\n\n",
            PROGRAM_NAME, outfile.name);
    }
    else
    {
        printf("\n%s: error encountered, file \"%s\" not completed\n\n",
            PROGRAM_NAME, outfile.name);
    }

    CloseFile(&outfile);
    CloseFile(&infile);

    /* destroy incomplete file if not configured elsewise */
    if(result != TRUE)
    {
        if(PRECIOUS == FALSE)
        {
            assert(out != NULL);
            remove(out);
        }
    }

    /* destroy the local options for these files */
    DestroyLocalOption();

    /* destroy the project store as well */
    DestroyVariableStore(&projectVarStore);

    /* destroy the ALT text store */
    DestroyVariableStore(&altTextVarStore);

    return result;
}

BOOL ProcessResponseFile(const char *resp)
{
    char textline[128];
    char defResp[MAX_PATHNAME_LEN];
    char newDirectory[MAX_PATHNAME_LEN];
    char oldDirectory[MAX_PATHNAME_LEN];
    TEXTFILE respfile;
    int result;
    char *in;
    char *out;
    char *ptr;
    BOOL useNewDir;
    BOOL respFileOpen;
    FIND_TOKEN findToken;

    assert(resp != NULL);

    useNewDir = FALSE;

    if(strchr(ALL_FILESYSTEM_DELIMITERS, resp[strlen(resp) - 1]) != NULL)
    {
        /* some tests as done to ensure that (a) newDirectory does not trail */
        /* with a directory separator and that (b) the separator is present */
        /* before appending the filename ... requirement (a) is a DOS issue */

        /* the response file is actually a directory the response file is */
        /* possibly kept in ... copy it to the newDirectory variable for */
        /* later use, but remove the trailing delimiter (MS-DOS issue) */
        strcpy(newDirectory, resp);
        newDirectory[strlen(newDirectory) - 1] = NUL;

        /* now, see if default response file is present */
        strcpy(defResp, newDirectory);
        strcat(defResp, DIR_DELIMITER_STRING);
        strcat(defResp, DEFAULT_RESPONSE_FILE);

        useNewDir = TRUE;

        respFileOpen = OpenFile(defResp, defResp, "r", &respfile);
    }
    else
    {
        respFileOpen = OpenFile(resp, resp, "r", &respfile);
    }

    if(respFileOpen == FALSE)
    {
        printf("%s: unable to open \"%s\" as a response file\n", PROGRAM_NAME,
            resp);
        return FALSE;
    }

    printf("%s: Processing response file \"%s\" ...\n", PROGRAM_NAME,
        respfile.name);

    /* processing a response file in another directory, change to that */
    /* directory before processing the files */
    if(useNewDir)
    {
        getcwd(oldDirectory, sizeof oldDirectory);
        chdir(newDirectory);
    }

    result = TRUE;
    do
    {
        if(GetFileLine(&respfile, textline, sizeof(textline)) == FALSE)
        {
            break;
        }

        in = NULL;
        out = NULL;

        /* walk tokens ... allow for tab character as token and ignore */
        /* multiple token characters between filenames */
        ptr = StringFirstToken(&findToken, textline, " \t");
        while(ptr != NULL)
        {
            /* is this just a repeated token? */
            if((*ptr == ' ') || (*ptr == '\t'))
            {
                ptr = StringNextToken(&findToken);
                continue;
            }

            /* found something ... like parsing the command-line, look for */
            /* options, then response files, then regular in and out filenames */
            if((*ptr == '-') || (*ptr == '/'))
            {
                /* option */
                ParseTextOption(ptr, OptionCallback, 0);
            }
            else if(*ptr == ';')
            {
                /* comment, ignore the rest of the line */
                break;
            }
            else if(in == NULL)
            {
                in = ptr;
            }
            else if(out == NULL)
            {
                out = ptr;
            }
            else
            {
                /* hmm ... extra information on line */
                HtpMsg(MSG_WARNING, &respfile, "extra option \"%s\" specified in response file, ignoring",
                    ptr);
            }

            ptr = StringNextToken(&findToken);
        }

        /* if in and out NULL, ignore the line entirely (all options or blank) */
        if((in == NULL) && (out == NULL))
        {
            continue;
        }

        if(out == NULL)
        {
            /* in is the response file ... recurse like theres no tomorrow */
            result = ProcessResponseFile(in);
            continue;
        }

        /* both in and out were specified, do it */
        result = ProcessFileByName(in, out);
    } while(result == TRUE);

    CloseFile(&respfile);

    /* restore the directory this all started in */
    if(useNewDir)
    {
        chdir(oldDirectory);
    }

    return result;
}   

BOOL ProcessDefaultFile(void)
{
    TASK defTask;
    TEXTFILE infile;
    TEXTFILE outfile;
    BOOL result;

    /* get the default filename */
    if(HtpDefaultFilename(globalFilename, MAX_PATHNAME_LEN) == FALSE)
    {
        /* nothing to do, but no error either */
        globalFilename[0] = NUL;
        return TRUE;
    }

    if(OpenFile(globalFilename, globalFilename, "r", &infile) == FALSE)
    {
        printf("%s: unable to open default file \"%s\"\n", PROGRAM_NAME,
            globalFilename);
        return FALSE;
    }

    /* use a null outfile because there is no file to write to */
    CreateNullFile(&outfile);

    /* build a task (just like any other) and process the file */
    /* use the global variable store to hold all the macros found */
    defTask.infile = &infile;
    defTask.outfile = &outfile;
    defTask.varstore = &globalVarStore;
    defTask.sourceFilename = globalFilename;

    printf("%s: Processing default file \"%s\" ... \n", PROGRAM_NAME,
        globalFilename);

    result = ProcessTask(&defTask);

    CloseFile(&infile);
    CloseFile(&outfile);

    return result;
}

int main(int argc, char *argv[])
{
    int result;
    uint ctr;
    char *in;
    char *out;
    char *resp;

    DisplayHeader();

    if(argc == 1)
    {
        usage();
        return 1;
    }

    /* initialize debugging */
#if DEBUG
    DebugInit("htpdeb.out");
    atexit(DebugTerminate);
#endif

    /* initialize the suballoc memory module */
    InitializeMemory();
    atexit(TerminateMemory);

    /* initialize global variable options */
    if(InitializeGlobalOption(OptionCallback, 0) == FALSE)
    {
        printf("%s: fatal error, unable to initialize internal options\n",
            PROGRAM_NAME);
        return 1;
    }

    in = NULL;
    out = NULL;
    resp = NULL;

    /* search command-line for options */
    for(ctr = 1; ctr < (uint) argc; ctr++)
    {
        if((*argv[ctr] == '-') || (*argv[ctr] == '/'))
        {
            /* command-line option specified */
            ParseTextOption(argv[ctr], OptionCallback, 0);
        }
        else if(*argv[ctr] == '@')
        {
            /* response file specified */
            resp = argv[ctr] + 1;
            if(*resp == NUL)
            {
                resp = (char *) DEFAULT_RESPONSE_FILE;
            }
        }
        else if(in == NULL)
        {
            /* input file was specified */
            in = argv[ctr];
        }
        else if(out == NULL)
        {
            /* output file was specified */
            out = argv[ctr];
        }
        else
        {
            printf("%s: unknown argument \"%s\" specified\n",
                PROGRAM_NAME, argv[ctr]);
            return 1;
        }
    }
    
    if(USAGE == TRUE)
    {
        usage();
        return 1;
    }

    if((in == NULL || out == NULL) && (resp == NULL))
    {
        usage();
        return 1;
    }

    /* initialize the global variable store before proceeding */
    if(InitializeVariableStore(&globalVarStore) != TRUE)
    {
        printf("%s: unable to initialize global variable store (out of memory?)\n",
            PROGRAM_NAME);
        return 1;
    }

    /* before reading in the response file or processing any files, handle */
    /* the default file, if there is one ... all of its macros are held in */
    /* the global variable store */
    ProcessDefaultFile();

    /* now, process the response file (if there is one) or the files */
    /* specified on the command-line */
    if(resp != NULL)
    {
        result = ProcessResponseFile(resp);
    }
    else
    {
        result = ProcessFileByName(in, out);
    }

    /* display varstore stats */
    DEBUG_PRINT(("Variable lookups=%u  string cmps=%u  missed string cmps=%u  cache hits=%u  hash calcs=%u\n",
        variableLookups, variableStringCompares, variableMissedStringCompares,
        variableCacheHits, variableHashCalcs));

    /* display macro expansion stats */
    DEBUG_PRINT(("Expansion skipped=%u  performed=%u\n", expandSkipped,
        expandPerformed));

    /* destroy the global variable store */
    DestroyVariableStore(&globalVarStore);

    /* destroy global option */
    DestroyGlobalOption();

    /* display suballoc stats */
    DEBUG_PRINT(("suballoc  total allocations=%u  free pool hits=%u  system heap allocs=%u\n",
        totalAllocations, freePoolHits, totalAllocations - freePoolHits));

    return (result == TRUE) ? 0 : 1;
}