/*
 *
 *   Copyright (c) International Business Machines  Corp., 2000
 *
 *   This program is free software;  you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation; either version 2 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY;  without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
 *   the GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program;  if not, write to the Free Software
 *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 *
 * Module: scanner.c
 */

/*
 * Change History:
 *
 */

/*
 *
 */

/* This module implements the scanner, which is the front end of the syntactic
   analyzer used by the LVM command.  The scanner is an FSA whose states and
   actions are as indicated in the table below:
                                                                Token Type
      State   Characters that trigger a           State to         Output To
   Name           Transition                   Transition to    Screener
   ----------------------------------------------------------------------------

   Start  --
          ' '                               -> SingleSpace
          '\t'                              -> SingleTab
          ',',':','{','}' , '=' , '(' , ')' -> Start                    => "Separator"
          '-'                               -> OptionCheck
          '+'                               -> PositiveNumberCheck
          '0' .. '9'                        -> IsNumber
          '"'                               -> IsString
          '/'                               -> IsPathName
          'A' .. 'Z' , 'a' .. 'z'           -> IsKeyWord
          EOF                               -> EndState
                                            -> Error;

   OptionCheck --
          '-'                               -> Start                    => "Separator"
                                            -> IsNegativeNumber;

   IsNegativeNumber --
          '0' .. '9'                        -> IsNumber
                                            -> Start                    => "Separator";

   IsNumber --
          '0' .. '9'                        -> IsNumber
          '.'                               -> IsRealNumber
          'A' .. 'Z', 'a' .. 'z'            -> IsKeyWord                => "Number"
                                            -> SeparatorOrError         => "Number";

   IsRealNumber --
          '0' .. '9'                        -> IsRealNumber
          'A' .. 'Z', 'a' .. 'z'            -> IsKeyWord                => "RealNumber"
                                            -> SeparatorOrError         => "RealNumber";

   SeparatorOrError --
          ',',':','}','(',')'               -> Start                    => "Separator"
          ' '                               -> SingleSpace
          '\t'                              -> SingleTab
          EOF                               -> EndState
                                            -> Error;

   PositiveNumberCheck --
          '0' .. '9'                        -> IsNumber
                                            -> EndState                 => "Invalid Character";

   IsPathName --
          '/' , 'A' .. 'Z' , 'a' .. 'z'     -> IsPathName
          '0' .. '9'                        -> IsPathName
          '_' , '-'                         -> IsPathName
                                            -> SeparatorOrError         => "String";
   
   IsKeyWord --
          'A' .. 'Z' , 'a' .. 'z'           -> IsKeyWord
          '0' .. '9'                        -> IsKeyWord
          '_' , '-'                         -> IsKeyWord
          '/'                               -> IsPathName
                                            -> SeparatorOrError         => "Keyword";

   EndOfString --
          '"'                               -> IsString
                                            -> Start                    => "String";
   IsString --
          '"'                               -> EndOfString
          accept any character other than
            the single quote mark as being
            part of the string              -> IsString;

   SingleSpace --
          ' '                               -> IsMultiSpace
                                            -> Start                    => "Space";

   IsMultiSpace --
          ' '                               -> IsMultiSpace
                                            -> Start                    => "MultiSpace";

   SingleTab --
          '\t'                              -> IsMultiTab
                                            -> Start                    => "Tab";

   IsMultiTab --
          ' '                               -> IsMultiTab
                                            -> Start                    => "MultiTab";

   Error --
                                            -> EndState                 => "ERROR";

   EndState --
                                                                        => "EOF";

   ----------------------------------------------------------------------------

   The scanner maintains a buffer.  Each time a character is used in a
   transition, it is placed into the buffer.  The buffer is cleared each
   time a transition to the Start state is made.  When the scanner reaches
   a state where it outputs a value (as indicated in the table), the output
   consists of two parts: the contents of the buffer, and a characterization
   of the contents of the buffer.  In the table above, only the characterization
   is shown in the output column.  In those cases where output occurs on a
   transition to the start state, the output takes place before the transition
   to the start state.  Each of the items "output" by the scanner is appended
   to a linked list, which is returned to the caller when scanning has been
   completed.  Thus, the scanner returns a linked list of tokens.                 */


/* Identify this file. */
#define SCANNER_C

/*--------------------------------------------------
 * Necessary include files
 --------------------------------------------------*/
#include <ctype.h>    /* toupper */
#include <stdlib.h>   /* malloc */
#include <stdio.h>
#include <string.h>
#include "token.h"    /* TokenType, TokenCharacterizations, MaxIdentifierLength */
#include "error.h"    /* Scanner_Errors, Report_Scanner_Error */
#include "scanner.h"  /* GetToken, SetInput prototypes */

/*--------------------------------------------------
 * Private constants
 --------------------------------------------------*/



/*--------------------------------------------------
 * Private Type definitions
 --------------------------------------------------*/

/* The following enumeration has one entry for each state in the
   state table.  A variable of this type will be used to keep
   track of which state the FSA is in at any given time.          */
typedef enum
{
  Start,
  OptionCheck,
  IsNegativeNumber,
  IsNumber,
  IsRealNumber,
  SeparatorOrError,
  PositiveNumberCheck,
  IsKeyWord,
  IsPathName,
  EndOfString,
  IsString,
  SingleSpace,
  IsMultiSpace,
  SingleTab,
  IsMultiTab,
  EndState,
  ErrorState
} State;


/*--------------------------------------------------
  Private global variables.
--------------------------------------------------*/
static char *  CommandLine;                       /* Original command line passed to us. */
static char *  CurrentCommandLine;                /* Starting point within the command line passed to us. */
static uint    CharactersConsumed;                /* Count of the number of characters we have used from the command line. */
static uint    CurrentRow;                        /* If the command line is actually a command file, this will indicate which line of the command file is being processed. */
static uint    CurrentColumn;                     /* The current position (0 based) within the current line/command line expressed as an offset. */
static FILE *  InputFile;                         /* The command file being processed, if there is one. */
static char    Buffer[MaxIdentifierLength];       /* Our buffer for creating tokens. */
static uint    PositionInBuffer;                  /* Used to keep track of where to put characters in the Buffer. */
static uint    Flush_Count = 1;                   /* Used by the GetCharacter function to advance past the end of comments. */
static BOOLEAN ExamineCurrentCharacter = FALSE;   /* Causes a state to not load a new character but to examine the existing one. */
static char    CharacterToExamine = ' ';          /* The current character to examine.  Examination of a character will
                                                     result in the FSA transitioning to a new state in accordance with
                                                     the state table at the beginning of this file.                              */
static char    NextCharacter = ' ';               /* The next character that will be examined. */
static BOOLEAN Init_Characters = TRUE;            /* Controls the initialization of NextCharacter and CharacterToExamine. */
static BOOLEAN Detect_Comments = TRUE;            /* Control comment detection and elimination.  If FALSE, then comment detection is disabled. */
static State   CurrentState = Start;              /* The current state within the FSA.                                           */
static char *  Error_Source = NULL;               /* Used for error reporting purposes.  It points to either CommandLine or the name of the input file. */


/*--------------------------------------------------
 Local Function Prototypes
--------------------------------------------------*/

/* The following function creates a token using the the characters in the Buffer.  */
static TokenType * MakeToken( TokenCharacterizations Characterization);

/* The following function gets a character from the command line or the input file.
   It adjusts the CharactersConsumed and CurrentColumn variables everytime it gets
   a character, and it adjusts both the CurrentColumn and CurrentRow variables whenever
   it finds a carriage return or carriage return/linefeed pair.
   NOTE:  If GetCharacter encounters a NULL in a file or command line, it will treat
          that as EOF.  In fact, GetCharacter will return NULL to represent EOF.
   NOTE:  If there is an I/O error during input from a command file, this will be
          treated as EOF also and NULL will be returned for the character value.
   NOTE:  Whenever EOF is returned by this function, if the source of input is a command
          file, the command file will be closed.                                         */
static void GetCharacter( void );

/* KeepCharacter - This puts the character passed to it into Buffer if Buffer is not yet full. */
static void KeepCharacter( void );

/*--------------------------------------------------
 There are no public global variables.
--------------------------------------------------*/



/*--------------------------------------------------
 * Public Functions Available
 --------------------------------------------------*/

/*********************************************************************/
/*                                                                   */
/*   Function Name: SetInput                                         */
/*                                                                   */
/*   Descriptive Name: Sets the input source for drawing characters  */
/*                     used to build tokens.                         */
/*                                                                   */
/*   Input: BOOLEAN IsFile - if TRUE, then the following parameter   */
/*                           is interpreted as the name of a file to */
/*                           be used for input.                      */
/*          char * FilenameOrString - If IsFile is TRUE, then this   */
/*                           is the name of the file to use for      */
/*                           input.  If IsFile is FALSE, then this   */
/*                           is a pointer to a buffer containing a   */
/*                           NULL terminated string which will be    */
/*                           used as input for building tokens.      */
/*                                                                   */
/*   Output: The function returns TRUE if it succeeded, FALSE        */
/*           otherwise.                                              */
/*                                                                   */
/*   Error Handling:                                                 */
/*                                                                   */
/*   Side Effects:                                                   */
/*                                                                   */
/*   Notes:                                                          */
/*                                                                   */
/*********************************************************************/
BOOLEAN SetInput(BOOLEAN IsFile, char * FilenameOrString)
{

  uint Index;   /* Used to walk Buffer. */

  /* Initialize our global variables. */
  CommandLine = NULL;
  CurrentCommandLine = NULL;
  CharactersConsumed = 0;
  CurrentRow = 1;
  CurrentColumn = 0;
  InputFile = NULL;
  CurrentState = Start;
  Error_Source = NULL;
  PositionInBuffer = 0;
  ExamineCurrentCharacter = FALSE;
  Flush_Count = 1;
  CharacterToExamine = ' ';
  NextCharacter = ' ';

  /* Null out Buffer. */
  for (Index = 0; Index < MaxIdentifierLength; Index++)
    Buffer[Index] = 0;

  if (!IsFile)
  {
    /* Since we have a command line, lets save it accordingly. */
    CommandLine = FilenameOrString;
    CurrentCommandLine = CommandLine;
    Error_Source = CommandLine;

    /* Indicate success. */
    return TRUE;

  }
  else
  {
    /* Since we have a command file, we must open it and prepare it for use. */

    /* Open the file. */
    InputFile = fopen(FilenameOrString, "rt");

    /* Did we succeed? */
    if ( InputFile )
    {
      Error_Source = strdup(FilenameOrString);   
      if ( Error_Source == NULL )
      {
        /* Report the error. */
        Report_Scanner_Error(Scanner_Out_Of_Memory,NULL,NULL,0,0);

        /* Indicate failure. */
        return FALSE;
        
      }

      return TRUE;          /* Indicate success */
    }
    else
    {

      /* Report the error. */
      Report_Scanner_Error(Bad_Command_File, FilenameOrString,NULL,0,0);

      /* Indicate failure. */
      return FALSE;

    }

  }

}


/*********************************************************************/
/*                                                                   */
/*   Function Name: GetToken                                         */
/*                                                                   */
/*   Descriptive Name: Returns a token derived from the source set   */
/*                     by the SetInput function.                     */
/*                                                                   */
/*   Input: None.                                                    */
/*                                                                   */
/*   Output: The function return value is a token.                   */
/*                                                                   */
/*   Error Handling:                                                 */
/*                                                                   */
/*   Side Effects: May alter the following static global variables,  */
/*                 either directly or by calling other functions:    */
/*                   CurrentCommandLine                              */
/*                   CharactersConsumed                              */
/*                   CurrentRow                                      */
/*                   CurrentColumn                                   */
/*                   InputFile                                       */
/*                   Buffer                                          */
/*                   PositionInBuffer                                */
/*                   ExamineCurrentCharacter                         */
/*                   CharacterToExamine                              */
/*                   CurrentState                                    */
/*                                                                   */
/*   Notes:                                                          */
/*                                                                   */
/*********************************************************************/
TokenType * GetToken(void)
{
  /* The FSA depicted in the state table at the beginning of this file is
     simulated using a switch statement.  Each case in the switch statement
     corresponds to a state in the state table.  The CurrentState variable
     is used to indicate which state the FSA is in.                         */

  for (;;)
  {
    switch (CurrentState)
    {
      case Start :          /* The START state */

        /* Initialize PositionInBuffer to 0 since we are beginning a new token. */
        PositionInBuffer = 0;

        /* Enable comment detection and elimination. */
        Detect_Comments = TRUE;

        /* Get a character to examine.*/
        GetCharacter();

        /* Are we at EOF? */
        if (CharacterToExamine == 0 )
        {

          /* We are out of input.  Go to the EndState. */
          CurrentState = EndState;
          break;

        }

        /* If the first character is a quote, then we will not be keeping it. */
        if ( CharacterToExamine == 0x22 )
        {

          /* Comments can not appear in strings, so disable comment detection. */
          Detect_Comments = FALSE;

          /* Go to the IsString state. */
          CurrentState = IsString;
          break;

        }

        /* Since we will be keeping this character, put it in the buffer. */
        KeepCharacter();

        /* Process the current character according to the state table. */

        /* Check for a space. */
        if (CharacterToExamine == ' ')
        {
          CurrentState = SingleSpace; /* Transition to the SingleSpace state. */
          break;
        }

        /* Check for a tab. */
        if (CharacterToExamine == '\t')
        {
          CurrentState = SingleTab; /* Transition to the SingleTab state. */
          break;
        }

        /* Check for a minus sign. */
        if ( CharacterToExamine == '-' )
        {

          /* Transition to the OptionCheck state. */
          CurrentState = OptionCheck;
          break;

        }

        /* Check for a plus sign. */
        if ( CharacterToExamine == '+' )
        {

          /* Transition to the PositiveNumberCheck state. */
          CurrentState = PositiveNumberCheck;
          break;

        }

        /* Check for a number. */
        if ( ( CharacterToExamine >= '0' ) &&
             ( CharacterToExamine <= '9' )
           )
        {

          /* Transition to the IsNumber state. */
          CurrentState = IsNumber;
          break;

        }

        /* Check for a path name. */
        if ( CharacterToExamine == '/' )
        {

          /* Transition to the IsPathName state. */
          CurrentState = IsPathName;
          break;
          
        }

        /* Is the character a letter of the alphabet? */
        if ( (
               ( CharacterToExamine >= 'A' ) &&
               ( CharacterToExamine <= 'Z' )
             ) ||
             (
               ( CharacterToExamine >= 'a' ) &&
               ( CharacterToExamine <= 'z' )
             )
           )
        {

          /* Transition to the IsKeyword state. */
          CurrentState = IsKeyWord;
          break;

        }

        /* Check for a separator. */
        if ( (CharacterToExamine == ',')
             ||
             (CharacterToExamine == ':')
             ||
             (CharacterToExamine == '{')
             ||
             (CharacterToExamine == '}')
             ||
             (CharacterToExamine == '(')
             ||
             (CharacterToExamine == ')')
             ||
             (CharacterToExamine == '=')
           )
        {

          /* We must create and return a separator token. */
          return MakeToken(Separator);

        }

        /* Since we did not recognize the character, go to the error state! */
        CurrentState = ErrorState;

        /* Report the error. */
        Report_Scanner_Error(Invalid_Character,Error_Source,&CharacterToExamine, CurrentColumn, CurrentRow);

        break;

      case OptionCheck:

        /* Get a character to examine. */
        GetCharacter();

        /* Is the character part of a number or something else? */
        if ( CharacterToExamine == '-' )
        {

          /* Keep the character. */
          KeepCharacter();

          /* Set the state to resume the FSA at the next time this function is called. */
          CurrentState = Start;

          /* We will treat this character as a separator since it is not part of a number. */
          return MakeToken(Separator);

        }

        /* Since we did not recognize the character, we will give the IsNegativeNumber state a chance. */
        ExamineCurrentCharacter = TRUE;
        CurrentState = IsNegativeNumber;
        break;

      case IsNegativeNumber:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the current character being examined a number? */
        if ( ( CharacterToExamine >= '0' ) && ( CharacterToExamine <= '9' ) )
        {

          /* We have a negative number!  Keep the character and transition to the IsNumber state. */
          KeepCharacter();
          CurrentState = IsNumber;
          break;

        }

        /* Since we did not recognize this character, it can't be part of a number.  We should only
           have a '-' in the buffer, so we will save the current character for the Start state
           to work on while we output a Separator token for the '-' in the buffer.                     */
        ExamineCurrentCharacter = TRUE;

        CurrentState = Start;

        return MakeToken(Separator);

        break; /* Keep the compiler happy. */

      case IsNumber:        /* The IsNumber state. */

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character to examine part of a number? */
        if ( (CharacterToExamine >= '0') && (CharacterToExamine <= '9') )
        {

          /* Lets keep the character. */
          KeepCharacter();

          break;

        }
        else
        {

          /* Is the character a period?  If so, we may have a real number. */
          if ( CharacterToExamine == '.' )
          {

            /* We will keep the character and change state. */
            KeepCharacter();

            CurrentState = IsRealNumber;
            break;

          }
          else
          {

            /* Is the character a letter of the alphabet? */
            if ( (
                   ( CharacterToExamine >= 'A' ) &&
                   ( CharacterToExamine <= 'Z' )
                 ) ||
                 (
                   ( CharacterToExamine >= 'a' ) &&
                   ( CharacterToExamine <= 'z' )
                 )
               )
            {

              /* Transition to the IsKeyword state. */
              CurrentState = IsKeyWord;

            }

          }

        }

        /* If CurrentState is still IsNumber, then we did not recognize the character.
           Since we did not recognize the character, transition to the SeparatorOrError state. */
        if ( CurrentState == IsNumber )
          CurrentState = SeparatorOrError;

        /* We want the next state to examine this token. */
        ExamineCurrentCharacter = TRUE;

        /* According to the FSA table at the beginning of this file, we must output a token. */
        return MakeToken(Number);

        break;

      case IsRealNumber:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character to examine part of a number? */
        if ( (CharacterToExamine >= '0') && (CharacterToExamine <= '9') )
        {

          /* Lets keep the character. */
          KeepCharacter();

          break;

        }

        /* Is the character a letter of the alphabet? */
        if ( (
               ( CharacterToExamine >= 'A' ) &&
               ( CharacterToExamine <= 'Z' )
             ) ||
             (
               ( CharacterToExamine >= 'a' ) &&
               ( CharacterToExamine <= 'z' )
             )
           )
        {

          /* Transition to the IsKeyword state. */
          CurrentState = IsKeyWord;

        }
        else
          /* Since we did not recognize the character, transition to the SeparatorOrError state. */
          CurrentState = SeparatorOrError;

        /* We want the next state to examine this token. */
        ExamineCurrentCharacter = TRUE;

        /* According to the FSA table at the beginning of this file, we must output a token. */
        return MakeToken(RealNumber);

        break;

      case SeparatorOrError:

        /* Get the next character to examine. */
        GetCharacter();

        /* Check for a separator. */
        if ( (CharacterToExamine == ',')
             ||
             (CharacterToExamine == ':')
             ||
             (CharacterToExamine == '{')
             ||
             (CharacterToExamine == '}')
             ||
             (CharacterToExamine == '(')
             ||
             (CharacterToExamine == ')')
             ||
             (CharacterToExamine == '=')
           )
        {

          /* We will keep the character and return to the start state after creating a token. */
          KeepCharacter();

          CurrentState = Start;

          /* We must create and return a separator token. */
          return MakeToken(Separator);

        }

        /* Check for a space. */
        if ( CharacterToExamine == ' ' )
        {

          /* Go to the SingleSpace state. */
          CurrentState = SingleSpace;

          /* Keep the current character. */
          KeepCharacter();

          break;

        }

        /* Check for a tab character. */
        if ( CharacterToExamine == '\t' )
        {

          /* Go to the SingleTab state. */
          CurrentState = SingleTab;

          /* Keep the current character. */
          KeepCharacter();

          break;

        }

        /* Check for EOF */
        if ( CharacterToExamine == 0 )
        {

          /* Go to the end state. */
          CurrentState = EndState;

          break;

        }

        /* Since we did not recognize the character, report the error and go to the error state! */
        KeepCharacter();

        Report_Scanner_Error(Invalid_Character, Error_Source, &CharacterToExamine, CurrentColumn, CurrentRow);

        CurrentState = ErrorState;
        break;

      case PositiveNumberCheck:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character to examine part of a number? */
        if ( (CharacterToExamine >= '0') && (CharacterToExamine <= '9') )
        {

          /* Lets keep the character. */
          KeepCharacter();

          /* Lets go to the IsNumber state. */
          CurrentState = IsNumber;

          break;

        }

        /* Since we did not recognize the character, report the error and go to the error state! */
        KeepCharacter();

        Report_Scanner_Error(Invalid_Character, Error_Source, &CharacterToExamine, CurrentColumn, CurrentRow);

        CurrentState = ErrorState;
        break;

      case IsPathName:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character a letter of the alphabet, a number, or '_' or '-'? */
        if ( ( CharacterToExamine == '/' ) ||
             (
               ( CharacterToExamine >= 'A' ) &&
               ( CharacterToExamine <= 'Z' )
             ) ||
             (
               ( CharacterToExamine >= 'a' ) &&
               ( CharacterToExamine <= 'z' )
             ) ||
            (
               ( CharacterToExamine >= '0' ) &&
               ( CharacterToExamine <= '9' )
            ) ||
            ( CharacterToExamine == '_' ) ||
            ( CharacterToExamine == '-' )
           )
        {

          /* Lets keep it. */
          KeepCharacter();

          break;

        }

        /* Since we did not recognize the character, transition to the SeparatorOrError state. */
        CurrentState = SeparatorOrError;

        /* We want the SeparatorOrError state to examine this token. */
        ExamineCurrentCharacter = TRUE;

        /* According to the FSA table at the beginning of this file, we must output a token. */
        return MakeToken(String);

        break;

      case IsKeyWord:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character a letter of the alphabet, a number, or '_' or '-'? */
        if ( 
             (
               ( CharacterToExamine >= 'A' ) &&
               ( CharacterToExamine <= 'Z' )
             ) ||
             (
               ( CharacterToExamine >= 'a' ) &&
               ( CharacterToExamine <= 'z' )
             ) ||
            (
               ( CharacterToExamine >= '0' ) &&
               ( CharacterToExamine <= '9' )
            ) ||
            ( CharacterToExamine == '_' ) ||
            ( CharacterToExamine == '-' )
           )
        {

          /* Lets keep it. */
          KeepCharacter();

          break;

        }

        /* Is the character a '/'?  If so, transition to the IsPathName state. */
        if ( CharacterToExamine == '/' )
        {
          /* Let the IsPathName state handle this character. */
          ExamineCurrentCharacter = TRUE;
          CurrentState = IsPathName;
          break;
        }

        /* Since we did not recognize the character, transition to the SeparatorOrError state. */
        CurrentState = SeparatorOrError;

        /* We want the SeparatorOrError state to examine this token. */
        ExamineCurrentCharacter = TRUE;

        /* According to the FSA table at the beginning of this file, we must output a token. */
        return MakeToken(KeyWord);

        break;

      case EndOfString :


        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character the ending quote mark for the string?  If it is, then the Quote that 
           caused this state to be invoked is an embedded quote.  We will keep this quote and
           return to the IsString state to continue looking for the end of the string.             */
        if ( CharacterToExamine == 0x22 )
        {

          KeepCharacter();
          CurrentState = IsString;
          
        }
        else
        {

          /* We have found the end of the string.  The next state is the start state. */
          CurrentState = Start;

          /* We want the current character to be returned on the next GetCharacter call so that it
             can be examined by the Start state.                                                   */
          ExamineCurrentCharacter = TRUE;

          /* Output the token. */
          return MakeToken(String);

        }

        break;

      case IsString:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character the ending quote mark for the string?  Are we at EOF? */
        if ( ( CharacterToExamine == 0x22 ) ||
             ( CharacterToExamine == 0 )
           )
        {

          /* We may have found the end of the string. */
          CurrentState = EndOfString;

        }
        else
        {

          /* Keep the character and return to this state. */
          KeepCharacter();

        }

        break;

      case SingleSpace:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character a space? */
        if ( CharacterToExamine == ' ' )
        {

          /* Keep the character and go to the multi-space state. */
          KeepCharacter();

          CurrentState = IsMultiSpace;

          break;

        }

        /* Since the following character was not a space, set up to output a token and
           resume the FSA at the start state with the current character.                */
        ExamineCurrentCharacter = TRUE;
        CurrentState = Start;

        return MakeToken(Space);

        break;  /* Keep the compiler happy. */

      case IsMultiSpace:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character a space? */
        if ( CharacterToExamine == ' ' )
        {

          /* Keep the character and stay in this state. */
          KeepCharacter();

          break;

        }

        /* Since the following character was not a space, set up to output a token and
           resume the FSA at the start state with the current character.                */
        ExamineCurrentCharacter = TRUE;
        CurrentState = Start;

        return MakeToken(MultiSpace);

        break;  /* Keep the compiler happy. */

      case SingleTab:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character a tab? */
        if ( CharacterToExamine == '\t' )
        {

          /* Keep the character and go to the IsMultiTab state. */
          KeepCharacter();

          CurrentState = IsMultiTab;

          break;

        }

        /* Since the following character was not a tab, set up to output a token and
           resume the FSA at the start state with the current character.                */
        ExamineCurrentCharacter = TRUE;
        CurrentState = Start;

        return MakeToken(Tab);

        break;  /* Keep the compiler happy. */

      case IsMultiTab:

        /* Get the next character to examine. */
        GetCharacter();

        /* Is the character a tab? */
        if ( CharacterToExamine == '\t' )
        {

          /* Keep the character and stay in this state. */
          KeepCharacter();

          break;

        }

        /* Since the following character was not a tab, set up to output a token and
           resume the FSA at the start state with the current character.                */
        ExamineCurrentCharacter = TRUE;
        CurrentState = Start;

        return MakeToken( MultiTab );

        break;  /* Keep the compiler happy. */

      case EndState :       /* The "END" state. */

        /* Cleanup */
        if ( InputFile != NULL )
        {

          /* Since we were getting our input from a command file, close it. */
          fclose(InputFile);
          InputFile = NULL;

        }

        /* Make an EOF token. */
        return MakeToken(EofToken);

      case ErrorState :     /* The "ERROR" state. */

        /* Set up to go to the end state. */
        CurrentState = EndState;

        /* Now lets make an error token. */
        return MakeToken(InvalidCharacter);

        break;

    } /* switch */

  }  /* for */

}


/*--------------------------------------------------
 * Local Functions Available
 --------------------------------------------------*/


/*********************************************************************/
/*                                                                   */
/*   Function Name: MakeToken                                        */
/*                                                                   */
/*   Descriptive Name: Creates a token from the contents of the scan */
/*                     buffer and returns it as the function result. */
/*                                                                   */
/*   Input: TokenTypes Characterization - The characterization to    */
/*                                        assign to the token being  */
/*                                        made.                      */
/*                                                                   */
/*   Output: If Success : The function return value will be non-NULL,*/
/*                        and it will be a pointer to a new token.   */
/*                                                                   */
/*           If Failure : The function return value will be NULL.    */
/*                                                                   */
/*   Error Handling: If an error occurs, all memory allocated by this*/
/*                   function is freed, and NULL is returned as      */
/*                   the function return value.                      */
/*                                                                   */
/*   Side Effects:  Each position in buffer that contained a         */
/*                  character used in the token is set to NULL.      */
/*                                                                   */
/*   Notes:  None.                                                   */
/*                                                                   */
/*********************************************************************/
static TokenType * MakeToken(TokenCharacterizations Characterization)
{
  uint        Index;         /* Used as an index when stepping through the Buffer. */
  TokenType * New_Token;     /* The return value. */


  /* Do we have the memory for our return value? */
  New_Token = (TokenType *) malloc(sizeof(TokenType) );
  if ( New_Token == NULL)
  {

    Report_Scanner_Error(Scanner_Out_Of_Memory, Error_Source, &CharacterToExamine, CurrentColumn, CurrentRow);

    return NULL;

  }

  /* To make a token, we must allocate memory for the contents of
     Buffer and then copy the contents of Buffer.  Once this has been
     done, we need to set the Characterization, TokenRow, TokenColumn,
     and TokenLength fields.                                           */

  if (PositionInBuffer > 0)
  {
    /* Allocate memory. */
    New_Token->TokenText = (char *) malloc(PositionInBuffer + 1);
    if (New_Token->TokenText == NULL)
    {
      /* Malloc failed!  We must be out of memory. */
      Report_Scanner_Error(Scanner_Out_Of_Memory, Error_Source, &CharacterToExamine, CurrentColumn, CurrentRow);
      free(New_Token);
      return NULL;
    }

    /* Copy the contents of Buffer. */
    for (Index = 0; Index < PositionInBuffer; Index++)
    {
      New_Token->TokenText[Index] = Buffer[Index];
      Buffer[Index] = 0;
    }

    /* Make sure that the string we copied from Buffer is NULL terminated. */
    New_Token->TokenText[PositionInBuffer] = 0;

  }
  else /* Buffer is empty */
    New_Token->TokenText = NULL;

  /* Characterize the token. */
  New_Token->Characterization = Characterization;

  /* Save the token length and position. */
  New_Token->TokenRow = CurrentRow;
  New_Token->TokenColumn = CurrentColumn;
  New_Token->TokenLength = PositionInBuffer;
  New_Token->In_Lookahead_Queue = FALSE;

  /* Reset the position at which characters will be added to the buffer. */
  PositionInBuffer = 0;

  /* Indicate success! */
  return New_Token;

}

/* The following function gets a character from the command line or the input file.
   It adjusts the CharactersConsumed and CurrentColumn variables everytime it gets
   a character, and it adjusts both the CurrentColumn and CurrentRow variables whenever
   it finds a carriage return or carriage return/linefeed pair.
   NOTE:  If GetCharacter encounters a NULL in a file or command line, it will treat
          that as EOF.  In fact, GetCharacter will return NULL to represent EOF.
   NOTE:  If there is an I/O error during input from a command file, this will be
          treated as EOF also and NULL will be returned for the character value.
   NOTE:  Whenever EOF is returned by this function, if the source of input is a command
          file, the command file will be closed.                                         */
static void GetCharacter( void )
{

  BOOLEAN    In_Comment = FALSE;
  BOOLEAN    Do_Flush;
  uint       Nesting_Level = 0;   /* Used to track nested comments. */

  /* Should we return the currently loaded character or get a new one? */
  if ( ExamineCurrentCharacter )
  {

    ExamineCurrentCharacter = FALSE;

    if ( Detect_Comments )
    {

      /* Check for the start of a comment. */
      if ( ( CharacterToExamine == '/' ) && ( NextCharacter == '*' ) )
      {
        Nesting_Level++;
        In_Comment = TRUE;
      }

    }

    if ( ! In_Comment ) 
      return;

  }

  /* We will be in a loop until we get an acceptable character to return. */
  do
  {

    /* Initialize CharacterToExamine to NextCharacter.  Initialize NextCharacter to 0, which is our EOF indicator. */
    CharacterToExamine = NextCharacter;
    NextCharacter = 0;

    /* Are we reading from a file or from the command line?*/
    if (CommandLine != NULL )
    {

      /* We must get the next character from the command line. */
      NextCharacter = *CurrentCommandLine;

      /* If the character we are returning is NULL, we must NOT advance the CurrentCommandLine pointer! */
      if ( *CurrentCommandLine != 0 )
        CurrentCommandLine++;

    }
    else
    {

      /* Is there an open file for us to deal with? */
      if ( InputFile != NULL )
      {

        /* We are reading from a file.  Get the next character from the file. */
        if ( fread(&NextCharacter,1,1,InputFile) != 1 )
        {

          /* There was a problem!  We have reached the end of the input file or an I/O error perhaps. */
          fclose(InputFile);

          /* Clear out InputFile so that future calls to this function will return NULL. */
          InputFile = NULL;

          /* Set NextCharacter to EOF. */
          NextCharacter = 0;

        }

      }

    }

    /* Have we been initialized yet?  If not, then CharacterToExamine will be EOF and NextCharacter will hold the first character. */
    if ( ! Init_Characters )
    {

      /* Before we return a character, we adjust the CurrentRow and CurrentColumn variables.  If CharacterToExamine is a line feed
         or carriage return/line feed pair, then we must adjust CurrentRow and CurrentColumn accordingly.  If it is anything else,
         then we just adjust CurrentColumn.                                                                                         */
      if ( CharacterToExamine == '\n' )
      {

        CurrentColumn = 0;
        CurrentRow++;

      }
      else
        CurrentColumn++;

      /* If the character is not NULL, then we must adjust the CharactersConsumed variable. */
      if ( CharacterToExamine != 0 )
        CharactersConsumed++;
      
    }

    if ( Detect_Comments )
    {

      if ( In_Comment )
      {
        /* Check for the end of the comment. */
        if ( ( CharacterToExamine == 0 ) ||
             ( ( CharacterToExamine == '*' ) && ( NextCharacter == '/' ) )
           )
        {
          Nesting_Level--;
          if ( Nesting_Level == 0 )
          {
            In_Comment = FALSE;
            Flush_Count = 2;
          }

        }

      }

      /* Check for the start of a comment. */
      if ( ( CharacterToExamine == '/' ) && ( NextCharacter == '*' ) )
      {
        Nesting_Level++;
        In_Comment = TRUE;
        Init_Characters = FALSE;  /* We need this here in case a file begins with a comment.  Without this, our CurrentRow value will be off by the number of lines used by that initial comment. */
      }

    }

    if ( Flush_Count > 0 )
    {
      Do_Flush = TRUE;
      Flush_Count--;
    }
    else
      Do_Flush = FALSE;

  } while ( ( CharacterToExamine == '\r' ) || ( CharacterToExamine == '\n' ) || ( In_Comment ) || ( Do_Flush ) );

  Init_Characters = FALSE;

//  if ( Init_Characters )
//  {
//    
//    Init_Characters = FALSE;
//    GetCharacter();
//
//  }

  /* Return whatever character we have. */
  return;

}

/* KeepCharacter - This puts the character passed to it into Buffer if Buffer is not yet full. */
static void KeepCharacter( void )
{

  /* Do we have room in the Buffer? */
  if ( PositionInBuffer < MaxIdentifierLength )
  {

    /* Place the character into the buffer. */
    Buffer[PositionInBuffer] = CharacterToExamine;
    PositionInBuffer++;

  }

  return;

}