// Copyright (C) 2024 Arthur I. Schwarz
//
// This file is part of the Java SLIP library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

/**
 * @file    SlipLexer.h
 * @author  A. Schwarz
 * @date    November 22, 2024
 *
 */

using namespace std;

# include <string>
# include <sstream>
# include <bits/stdc++.h>
# include "SlipDatum.h"
# include "SlipErr.h"
# include "SlipLexer.h"
# include "SlipRead.h"
# include "SlipToken.h"

namespace slip {

   using StringBuilder = string ;
   using Descriptor    = SlipRead::Descriptor;

   /**
    * Alphabet classes. Each letter in the input alphabet is assigned a class.
    * When the current character is being input, it is mapped to one of these
    * classes and becomes an event triggering a transition in the Mealy Finite
    * State Machine.
    *
    * The alphabet classes have a token type cognate. The token type is returned
    * in the lexer token to the parser. The token type represents the parser view
    * of the lexer object.
    *
    * Note: The mapping facility is only used for delimiters (<, >, {, }, (, ), :)
    *       and is otherwise ignored. The token type EOF is used as a placehoder
    *       for unused mappings.
    */
   class mapClass : public SlipRead::Descriptor {
   public:
      SlipRead::Descriptor& desc;
      mapClass(const int val, const string name, SlipRead::Descriptor& desc) 
               : SlipRead::Descriptor(val, name), desc(desc) {}
      Descriptor& type() { return desc; }
   }; // class mapClass : Descriptor

   static const mapClass& eof= { 0, "END", SlipRead::END};   //!< End of File
   static const mapClass& spc= { 1, "spc", SlipRead::END};   //!< whitespace
   static const mapClass& zro= { 2, "zro", SlipRead::END};   //!< zero = {0}
   static const mapClass& oct= { 3, "oct", SlipRead::END};   //!< octal       number
   static const mapClass& dec= { 4, "dec", SlipRead::END};   //!< decimal     number
   static const mapClass& hex= { 5, "hex", SlipRead::END};   //!< hexadecimal number
   static const mapClass& alp= { 6, "alp", SlipRead::END};   //!< alphabetic character
   static const mapClass& lpr= { 7, "lpr", SlipRead::LPRN};  //!< left  parenthesis   '= {'
   static const mapClass& rpr= { 8, "rpr", SlipRead::RPRN};  //!< right parenthesis   '}'
   static const mapClass& lcr= { 9, "lcr", SlipRead::LCRL};  //!< left  curly brace   '{'
   static const mapClass& rcr= {10, "rcr", SlipRead::RCRL};  //!< right curly brace   '}'
   static const mapClass& lng= {11, "lng", SlipRead::LANG};  //!< left  angle bracket '<'
   static const mapClass& rng= {12, "rng", SlipRead::RANG};  //!< right angle bracket '>'
   static const mapClass& cln= {13, "cln", SlipRead::COLON}; //!< colon               ':'
   static const mapClass& squ= {14, "squ", SlipRead::END};   //!< single quote        "'"
   static const mapClass& dqu= {15, "dqu", SlipRead::END};   //!< double quote        '"'
   static const mapClass& sls= {16, "sls", SlipRead::END};   //!< forward slash       '/'
   static const mapClass& esc= {17, "esc", SlipRead::END};   //!< escape character    '\'
   static const mapClass& ast= {18, "ast", SlipRead::END};   //!< asterisk            '*'
   static const mapClass& plu= {19, "plu", SlipRead::END};   //!< plus sign           '+'
   static const mapClass& min= {20, "min", SlipRead::END};   //!< minus sign          '-'
   static const mapClass& per= {21, "per", SlipRead::END};   //!< period              '.'
   static const mapClass& x  = {22, "x"  , SlipRead::END};   //!< hexadecimal initiator 'x'
   static const mapClass& d  = {23, "d"  , SlipRead::END};   //!< double precision    'd'
   static const mapClass& e  = {24, "e"  , SlipRead::END};   //!< single precision    'e'
   static const mapClass& u  = {25, "u"  , SlipRead::END};   //!< unsigned suffix     'u'
   static const mapClass& c  = {26, "c"  , SlipRead::END};   //!< character suffix    'c'


    /**
    * Mapping table.
    *
    * This is a map from each of the 8-bit (256) ASCII letters into a class.
    *
    * Access to ASCII classes during lookup is done by map[X], where X is
    * any value that can be converted into an 8-bit value, which is then
    * used as an mapping index.
    *
    * All unicode letters whose 16-bit value is greater than 255 (\u00FF) are
    * treated as a member of the alphabetic (alp) class.
    */
   static const mapClass map[] = {
        //   00  01  02  03  04  05  06  07  08  09  0A  0B  0C  0D  0E  0F
      /*00*/ eof,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc
      /*10*/,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc,spc
      /*20*/,spc,alp,dqu,alp,alp,alp,alp,squ,lpr,rpr,ast,plu,alp,min,per,sls
      /*30*/,zro,oct,oct,oct,oct,oct,oct,oct,dec,dec,cln,spc,lng,spc,rng,spc
      /*40*/,alp,hex,hex,alp,d  ,e  ,hex,alp,alp,alp,alp,alp,alp,alp,alp,alp
      /*50*/,alp,alp,alp,alp,alp,u  ,alp,alp,x  ,alp,alp,alp,esc,alp,alp,alp
      /*60*/,spc,hex,hex,alp,d  ,e  ,hex,alp,alp,alp,alp,alp,alp,alp,alp,alp
      /*70*/,alp,alp,alp,alp,alp,u  ,alp,alp,x  ,alp,alp,lcr,alp,rcr,alp,spc
      /*80*/,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp
      /*90*/,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp
      /*A0*/,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp
      /*B0*/,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp
      /*C0*/,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp
      /*D0*/,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp
      /*E0*/,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp
      /*F0*/,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp,alp
   }; // mapClass map[]

   /**
    * State enumerated values.
    *
    * In this implementation, each state in the Mealy Finite State Machine (MFSM)
    * is given a standard name. The start state is 's00', and all subsequent
    * states are prefixed by 's' and given a 2-digit number, as in 's07'. This
    * standardization allows the construction of a shared enumerated value to
    * name the states. This shared numbering scheme does not apply to the semantics
    * for the states. The semantics for the MSFMs are different.
    */
                                                            //   states
   static const Descriptor& s00= { 0, "s00:start "};        //!< start
   static const Descriptor& s01= { 1, "s01:zero  "};        //!< zero found, could be octal/hex
   static const Descriptor& s02= { 2, "s02:oct   "};        //!< octal number
   static const Descriptor& s03= { 3, "s03:dec   "};        //!< decimal number
   static const Descriptor& s04= { 4, "s04:hex   "};        //!< hexadecimal number
   static const Descriptor& s05= { 5, "s05:sign  "};        //!< sign = {+|-} before decimal number
   static const Descriptor& s06= { 6, "s06:float "};        //!< floating point number
   static const Descriptor& s07= { 7, "s07:exp   "};        //!< exponent 0.0= {d|e}found
   static const Descriptor& s08= { 8, "s08:esign "};        //!< exponent sign
   static const Descriptor& s09= { 9, "s09:string"};        //!< string
   static const Descriptor& s10= {10, "s10:dquote"};        //!< double quote = {"}
   static const Descriptor& s11= {11, "s11:squote"};        //!< single quote = {'}
   static const Descriptor& s12= {12, "s12:endquo"};        //!< end quote
   static const Descriptor& s13= {13, "s13:'c'   "};        //!< 'c' found
   static const Descriptor& s14= {14, "s14:#U    "};        //!< integer 'U' found
   static const Descriptor& s15= {15, "s15:uchar "};        //!< 'c'U found

   /**
    *
    * In a FSM, given you are in one state, when an input event is
    * see GOTO the next state. The value in each cell in the table represents
    * the 'name' of the next state.
    *
    * For example, suppose you are processing a decimal number, if the next
    * input character is a number, stay in the same state, but if the next
    * input character is a space, then GOTO the beginning state and terminate
    * token processing. In this case the transitions would be:
    *
    * <pre><code>
    *                                                      oct
    *                                                 o-----------o
    *                                                 |    dec    |
    *                                                 |           |
    *                                                 v           |
    *                                           --------------    |
    *                                  o------->|    s03     | ---o
    *                                  |        --------------
    *       --------------    oct      |              |
    *       |    s00     |-------------o              |
    *       --------------    dec                     |
    *             ^                                   |
    *             |                                   |
    *             |               ~oct                |
    *             o-----------------------------------o
    *                             ~dec
    * </code></pre>
    *
    * where the arc transitions stand for<br>
    * oct<code>   0 1 2 3 4 5 6 7</code><br>
    * dec<code>   7 8</code><br>
    * ~<code>     </code>not, as in, not a octal number or decimal number
    */
   static const Descriptor GOTO[][16] = {
//                                     STATES
//                                      .        +/-   S   "S"  chr   '    U    U   
//             0   OCT  DEC  HEX  +/-  DBL  exp  exp  str  dqu  squ  squ       uin uchr
//events s00  s01  s02  s03  s04  s05  s06  s07  s08  s09  s10  s11  s12  s13  s14  s15
/*EOF*/ {s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s00}
/*spc*/,{s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s10, s12, s00, s00, s00, s00}
/*zro*/,{s01, s02, s02, s03, s04, s01, s06, s07, s08, s09, s10, s12, s09, s00, s09, s09}
/*oct*/,{s03, s02, s02, s03, s04, s03, s06, s07, s08, s09, s10, s12, s09, s00, s09, s09}
/*dec*/,{s03, s03, s00, s03, s04, s03, s06, s07, s08, s09, s10, s12, s09, s00, s09, s09}
/*hex*/,{s09, s09, s00, s00, s04, s09, s00, s00, s00, s09, s10, s12, s09, s00, s09, s09}
/*alp*/,{s09, s09, s00, s09, s09, s09, s00, s00, s00, s09, s10, s12, s09, s00, s09, s09}
/*lpr*/,{s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s10, s12, s00, s00, s00, s00}
/*rpr*/,{s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s10, s12, s00, s00, s00, s00}
/*lcr*/,{s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s10, s12, s00, s00, s00, s00}
/*rcr*/,{s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s10, s12, s00, s00, s00, s00}
/*lng*/,{s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s10, s12, s00, s00, s00, s00}
/*rng*/,{s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s10, s12, s00, s00, s00, s00}
/*cln*/,{s00, s00, s00, s00, s00, s00, s00, s00, s00, s00, s10, s12, s00, s00, s00, s00}
/*squ*/,{s11, s00, s00, s00, s00, s00, s00, s00, s00, s09, s10, s00, s13, s00, s00, s00}
/*dqu*/,{s10, s00, s00, s00, s00, s00, s00, s00, s00, s09, s00, s12, s09, s00, s00, s00}
/*sls*/,{s09, s00, s00, s00, s00, s00, s00, s00, s09, s09, s10, s12, s09, s00, s09, s09}
/*esc*/,{s09, s00, s00, s00, s00, s00, s00, s00, s00, s09, s10, s12, s09, s00, s09, s09}
/*ast*/,{s09, s00, s00, s00, s00, s00, s00, s00, s00, s09, s10, s12, s09, s00, s09, s09}
/*plu*/,{s05, s00, s00, s00, s00, s00, s08, s00, s08, s09, s10, s12, s09, s00, s09, s09}
/*min*/,{s05, s09, s00, s00, s00, s00, s00, s00, s08, s09, s10, s12, s09, s00, s09, s09}
/*per*/,{s06, s06, s06, s06, s00, s06, s00, s00, s00, s09, s10, s12, s09, s00, s09, s09}
/*x  */,{s09, s04, s00, s00, s09, s00, s00, s00, s00, s09, s10, s12, s09, s00, s09, s09}
/*d  */,{s09, s09, s00, s00, s04, s00, s07, s00, s00, s09, s10, s12, s09, s00, s09, s09}
/*e  */,{s09, s09, s00, s00, s04, s00, s07, s00, s00, s09, s10, s12, s09, s00, s09, s09}
/*u  */,{s09, s09, s00, s14, s00, s00, s00, s00, s00, s09, s10, s12, s09, s15, s09, s09}
      }; // Descriptor GOTO[][13]


   /**
    * Arc functions.
    *
    * At each FSM transition a function is executed. The values given stand for
    * the switch statement value which selects a case branch statement. This
    * brach statement performs the indicated operation.
    */
                                                            //   FSM function definition
  static const Descriptor& END= { 0, "EOF"};                //!< END OF FILE
  static const Descriptor& NUL= { 1, "NUL"};                //!< do nothing
  static const Descriptor& STP= { 2, "END"};                //!< generate an End of File  Token
  static const Descriptor& DEC= { 3, "DEC"};                //!< generate an  integer     Token
  static const Descriptor& ESC= { 4, "ESC"};                //!< Escape sequence processing
  static const Descriptor& BIN= { 5, "BIN"};                //!< process  a   decimal     Token
  static const Descriptor& CHR= { 6, "CHR"};                //!< character token, get next chr
  static const Descriptor& CTT= { 7, "CTT"};                //!< character token, keep chr
  static const Descriptor& DBL= { 8, "DBL"};                //!< generate a   double      Token
  static const Descriptor& HEX= { 9, "HEX"};                //!< process  a   hexadecimal Token
  static const Descriptor& OCT= {10, "OCT"};                //!< process  an  octal       Token
  static const Descriptor& SGN= {11, "SGN"};                //!< set negative sign
  static const Descriptor& STR= {12, "STR"};                //!< generate a   string      Token
  static const Descriptor& STT= {13, "STT"};                //!< string token, get next chr
  static const Descriptor& TOK= {14, "TOK"};                //!< return delimiter         Token
  static const Descriptor& UCK= {15, "UCK"};                //!< unsigned char token, keep chr
  static const Descriptor& UCH= {16, "UCH"};                //!< unsigned char token, get next chr
  static const Descriptor& UIK= {17, "UIK"};                //!< unsigned int token, keep chr
  static const Descriptor& UIN= {18, "UIN"};                //!< unsigned int token, get next chr
                                                         // diagnostic output
  static const Descriptor& TOO= {19, "TOO"};                //!< too many +|- signs
  static const Descriptor& EXP= {20, "EXP"};                //!< too many +|- signs in exponent
  static const Descriptor& NUM= {21, "NUM"};                //!< Illegal number - converting to string


   /**
    * FSM arc transition executable function.
    *
    * At each entry in the table, a function is executed (via a switch statement)
    * to execute. This function is the FSM arc transition function.
    *
    * For example in:
    *
    * <pre><code>
    *                                                      oct
    *                                                 o-----------o
    *                                                 |    dec    |
    *                                                 |           | --NUL
    *                                                 v           |
    *                                           --------------    |
    *                                  o------->|    s03     | ---o
    *                                  |        --------------
    *       --------------             |              |
    *       |    s00     |--- dec -----o -- NUL       |
    *       --------------                            | -- BIN
    *             ^                                   |
    *             |                                   |
    *             |               ~oct                |
    *             o-----------------------------------o
    *                             ~dec
    * </code></pre>
    *
    * where the arc transition functions are:<br>
    * NUL<code>   do nothing</code><br>
    * BIN<code>   generate a decimal number Token</code><br>
    *
    * Inputs to the MFSM consist of any legitmate write output from a SlipDatum
    * cell.
    *
    *
    *
    *
    *
    */
   static const Descriptor EXE[][16] = {
//                                     STATES
//                                      .        +/-   S   "S"  chr   '    U    U
//             0   OCT  DEC  HEX  +/-  DBL  exp  exp  str  dqu  squ  squ       uin uchr
//events s00  s01  s02  s03  s04  s05  s06  s07  s08  s09  s10  s11  s12  s13  s14  s15
/*EOF*/ {STP, DEC, DEC, DEC, DEC, CHR, DBL, STR, STR, STR, STT, CTT, CTT, CTT, UCH, UCK}
/*spc*/,{NUL, DEC, DEC, DEC, DEC, CHR, DBL, DBL, STR, STR, NUL, NUL, CHR, CHR, UIN, UCH}
/*zro*/,{NUL, NUL, OCT, BIN, HEX, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*oct*/,{BIN, OCT, OCT, BIN, HEX, BIN, NUL, NUL, NUL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*DEC*/,{BIN, OCT, DEC, BIN, HEX, BIN, NUL, NUL, NUL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*hex*/,{HEX, NUM, NUM, NUM, HEX, NUL, DBL, DBL, DBL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*alp*/,{NUL, NUM, NUM, NUM, NUM, NUL, DBL, DBL, DBL, NUL, NUL, NUL, CTT, CTT, NUL, NUL}
/*lpr*/,{TOK, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, STR, NUL, NUL, CTT, CTT, UCH, UCK}
/*rpr*/,{TOK, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, STR, NUL, NUL, CTT, CTT, UCH, UCK}
/*lcr*/,{TOK, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, STR, NUL, NUL, CTT, CTT, UCH, UCK}
/*rcr*/,{TOK, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, STR, NUL, NUL, CTT, CTT, UCH, UCK}
/*lng*/,{TOK, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, STR, NUL, NUL, CTT, CTT, UCH, UCK}
/*rng*/,{TOK, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, STR, NUL, NUL, CTT, CTT, UCH, UCK}
/*cln*/,{TOK, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, STR, NUL, NUL, CTT, CTT, UCH, UCK}
/*squ*/,{NUL, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, NUL, NUL, CHR, NUL, CTT, UCH, UCK}
/*dqu*/,{NUL, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, NUL, STT, NUL, NUL, CTT, UCH, UCK}
/*sls*/,{NUL, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*esc*/,{ESC, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, ESC, ESC, NUL, ESC, CTT, NUL, NUL}
/*ast*/,{NUL, DEC, DEC, DEC, DEC, CHR, DBL, DBL, DBL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*plu*/,{NUL, DEC, DEC, NUM, DEC, TOO, DBL, NUL, EXP, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*min*/,{SGN, DEC, DEC, NUM, DEC, TOO, DBL, NUL, EXP, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*per*/,{NUL, NUL, DEC, NUL, DEC, NUL, DBL, DBL, DBL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*x  */,{NUL, NUL, NUM, NUM, DEC, CHR, DBL, DBL, DBL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*d  */,{NUL, NUM, NUM, NUM, HEX, CHR, NUL, DBL, DBL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*e  */,{NUL, NUM, NUM, NUM, HEX, CHR, NUL, DBL, DBL, NUL, NUL, NUL, NUL, CTT, NUL, NUL}
/*u  */,{NUL, NUM, NUM, NUL, NUM, NUL, DBL, DBL, DBL, NUL, NUL, NUL, NUL, NUL, NUL, NUL}
  }; // Descriptor EXE[][13]


   /**
    * The lexer is responsible for separating the input text into tokens. The
    * available token types are described in the "Data Formats" table, in whUCH
    * each token can be recognized.
    * 
    * The lexer operates by accepting each input character in the ASCII stream,
    * determining what the character class by using the mapping array (map[char]),
    * and then doing a pattern search using the Mealy Finite State Machine.
    * The machine determines what to do with the mapped character class, both
    * how to process the character (EXE table)and how to continue pattern 
    * matching (GOTO table). When a pattern is successfully matched, a token
    * is created and returned to the parser.
    *
    * Each token is separated by a delimiter, as defined below. The parser is
    * responsible for determining whether an input token is legal within the
    * context of the parser structure (BNF) being analyzed. The lexer is a dumb
    * rendition of a input reader.
    *
    * The lexer is implemented as a Mealy Finite State Machine (FSM). A Finite State
    * Machine consists of nodes and arcs (see example below). A Mealy FSM has a
    * function associated with each arc. As a transition is made in the machine,
    * a function is executed. In this context, an input event is a label on an arc,
    * and the nodes represent a new state. We can then say that for any given- event
    * a function is performed and the FSM moves to a new state, where the new state
    * can be the same state or another state.
    *
    * FSM Example
    *
    * <pre><code>
    *                                           --------------
    *                                  o------->| state/node |
    *                                  |        --------------
    *       --------------    label    |
    *       | state/node |--- arc -----o
    *       --------------    event
    * </code></pre>
    *
    * The following definitions apply to the example:
    * <ul style="list-style-type:none">
    *    <li>state/node: The terms are used synonymously. A state represents a
    *                    place in the state table. Except for the start state,
    *                    a state always has an entering arc, all all states
    *                    have an exiting arc.</li>
    *    <li>label/event/arc: The terms are used synonymously. An exit from a
    *                    state is made when a input event is seen. Each event
    *                    corresponds to a arc label, where 'arc' is a generic
    *                    term representing the transition line connecting two
    *                    states.</li>
    * </ul>
    *
    * To represent this the lexer uses two tables, a GOTO table whUCH identifies
    * arc transitions (from node x to node y via some input event), and an EXE
    * table representing the 'arc' function to be performed.
    *
    * The lexer supports the Mealy FSM with two tables, a GOTO table, for state
    * transitions, and a EXE table to executable 'functions', whUCH in this case
    * is a switch statement labeled with a 'function' name. Both tables have the
    * form <code>table[event][state]</code>, where the 'event' is the arc
    * transition trigger, and the 'state' is the current state.
    *
    * Events in the lexer are alphabet classes, where a class is a grouping of
    * letters in the input alphabet whUCH are functionally related. For example,
    * the letters <code>0 1 2 3 4 5 6 7 8 9 A B C D E F</code> are members of the
    * hexadecimal class.
    *
    * >b>Lexical tokens</b>
    * Comments are not returned to the parser. There are two types of comments:<br>
    * <ul style="list-style-type:none">
    * <li><code>//</code>: line comment</li>
    * <li><code></code>block comment</li>
    * </ul>
    *
    * <table>
    *    <caption><center>Data Formats</center></caption>
    *    <th><center>data type</center></th>
    *    <th><center>format</center></th>
    *    <th><center>notes</center></th>
    *    <tr>
    *       <td>integer</td>
    *       <td>[+|-][0-9]+U?<br>
    *           0[0-7]+<br>
    *           0(x|X)[0-9a-fA-F]+</td>
    *       <td>decimal number<br>
    *           octal number<br>
    *           hexadecimal number</td>
    *    </tr>
    *    <tr>
    *       <td>floating point</td>
    *       <td>[+|-][0-9]+.[[e|E][ |+|-][0-9]+<br>
    *           [+|-][0-9]*.[0-9]+[[e|E][ |+|-][0-9]+<br>
    *           [+|-][0-9]+.[[d|D][ |+|-][0-9]+<br>
    *           [+|-][0-9]*.[0-9]+[[d|D][ |+|-][0-9]+</td>
    *       <td>single precision<br>
    *           single precision<br>
    *           double precision<br>
    *           double precision</td>
    *    </tr>
    *    <tr>
    *       <td>boolean</td>
    *       <td>true<br>
    *           false</td>
    *       <td>case ignored</td>
    *    </tr>
    *    <tr>
    *       <td>character</td>
    *       <td>c<br>
    *           'c'U?<br>
    *           '\''U?</td>
    *       <td>simple char<br>
    *           quoted char<br>
    *           escaped char</td>
    *    </tr>
    *    <tr>
    *       <td>string</td>
    *       <td>abc<br>
    *           "a.."<br>
    *           "... \c\" ...</td>
    *       <td>more than 1 char<br>
    *           quoted string</td>
    *           quoted " </tr>
    *    <tr>
    *       <td>escape sequence</td>
    *       <td>\'<br>
    *           \"<br>
    *           \\<br>
    *           \b<br>
    *           \f<br>
    *           \n<br>
    *           \r<br>
    *           \t
    *           \uFFFF</td>
    *       <td>single quote<br>
    *           double quote<br>
    *           backslash<br>
    *           backspace<br>
    *           form feed<br>
    *           line feed<br>
    *           carriage return<br>
    *           horizontal tab<br>
    *           unicode character</td>
    *    </tr>
    *
    * </table>
    *
    * Meta-characters:
    * <ul style="list-style-type:none">
    *    <li>[+|- ]: zero or one (+, -)</li>
    *    <li>[ ]+: one or more</li>
    *    <li>[ ]*: zero or more</li>
    *    <li>[ ]?: zero or one</li>
    *    <li>\c:  escape character</li>
    * </ul>
    *
    * Where one space is allowed, many are permitted.
    *
    * Delimiters serve to separate linguistic elements. The following
    * delimiters are used:
    * <pre>
    *    space<br>
    *    ( )      left/right parenthesis<br>
    *    { }      left/right curly braces<br>
    *    &lt; &gt;         left/right angle brackets<br>
    *    spc      space
    *    BOL      beginning of line<br>
    *    EOL      end of line<br>
    *    EOF      end of file/input
    * </pre>
    *
    * Text handling:
    * <ul style="list-style-type:none">
    *    <li>Character: A single character is defined as:</li>
    *       <ul style="list-style-type:dot">
    *          <li>a single character, as: a or b</li>
    *          <li>a single character in single quotes, as: 'a' or 'b'</li>
    *          <li>a single quoted escape sequence yielding a single character.
    *              SUIK as '\u00FF', '\n', '\12', '\0x12'. <\li>
    *       </ul>
    *    <li>String: two or more characters is defined as:</li>
    *       <ul style="list-style-type:dot">
    *          <li>two or more characters, as: cat or dog</li>
    *          <li>any number of characters in double quotes, as: "A"  or "cat" or
    *              "\n"</li>
    *          <li>an escape sequence contained in a double quoted string. SUIK
    *              as "abc\u00FF\t\0x00FF".</li>
    *       </ul>
    *    <li>Number: Any of the following number types:</li>
    *       <ul style="list-style-type:dot">
    *          <li>A floating point number. Treated as double precision.</li>
    *          <li>An integer. Treated as long.</li>
    *          <li>An octal number. Treated as long.</li>
    *          <li>A  hex number. Treated as long.</li>
    *       </ul>
    *    <li>Boolean: A boolean case insensitive value of true or false. Note that
    *        a true or false in a string ("true", "false") is treated as a string.</li>
    * </ul>
    *
    * Note that there are some anomalies:
    * <ol>
    *    <li>An escape sequence within a character or string is stored internally
    *        as binary value.</li>
    *    <li>A single quoted escape sequence larger than one byte is converted
    *        to a string. '0300', '\x300' are strings, not characters.</li>
    *    <li>Escape sequences not surrounded by quotes or double quotes is
    *        an integer. \n becomes an integer with value '10'.</li>
    *    <li>During input processing, if a number (decimal, octal, hexadecimal)
    *        is not terminated with a delimiter or space it is considered a string.
    *        For example, "10(" is a number followed by a left parentheses, but
    *        "10/" is a string.</li>
    *    <li>During input processing, if a number (decimal, octal, hexadecimal)
    *        contains an illegal character, it is treated as a string. For example,
    *        023A, 0x23AG, 3B are strings.</li>
    * </ol>
    *
    * Some examples:
    * <pre><code>
    *    0257        a octal       number, decimal value 175
    *    0x257       a hexadecimal number, decimal value 599
    *    257         a decimal     number, decimal value 257
    *    A           character
    *    'A'         character
    *    "A"         string
    *    0123A       string
    *    ++          string
    *    \n          character (line feed)
    *    '\n'        character (line feed)
    *    "\n"        string  (line feed)
    *    "str\012"   string  ("str\n")
    *    'ab'        string  ("ab")
    * </code></pre>
    *
    * @return token containing recognized type and value
    *
    * @author A. Schwarz
    */

   /**
    * Finite State Machine States.
    *
    * The values represent an index into the FSM tables, the GOTO and the
    * EXE tables, where <code>GOTO[alphabetic class][state]</code> represents
    * the transition arc to the next state in the FSM, and
    * <code>EXE[alphabetic class][state]</code> represents the function to be
    * performed at the transition.
    */

SlipToken& SlipLexer::lexer() {

   static const Descriptor& ENDLOOP = s00;                  //!< lexer loop end

   /******************* lexer code *******************/

      if (in.isEOF()) {
         token.start(in.getLine(), in.getCol());
         token.token(SlipRead::END);
         return token;
      };

      // remove all white space and comments before tokenization of input
      do {
         if (map[chr].value() == spc.value())
            while((chr = in.getChar()) == ' ');             // remove leading space

         //     Remove Comments
         if (chr == '/') {
            char peek = in.peek();
            if (peek == '/') {                              //!< line comment ("//")
               while( (chr = in.getChar()) != '\n' &&
                      (chr != in.END));
            } else if (peek == '*') {                       //!< block comment ("/* */")
               in.getChar();                                //   skip over '*'
               do {
                  while( (chr = in.getChar()) != '*' &&
                         (chr != in.END));
                  chr = in.getChar();
                  if ((chr == '/') || (chr == in.END)) break;
               } while (chr != in.END);
               chr = in.getChar();
            }
         }
      } while(map[chr].value() == spc.value());

      //    Token processing: Note we always start at s00
      Descriptor state = s00;                               //   lexer FSM start state
      int  sign  = 1;                                       //!< integer sign count
      unsigned int  value = 0;                              //!< integer value
      string str;                                           //!< local token copy

      token.start(in.getLine(), in.getCol());               //   create Token

      do {                                                  //   FSM loop
         mapClass clss = (chr > 0xFF)? alp: map[chr];       //!< character class
         Descriptor curState = state;
         str.push_back(chr);

         state          = GOTO[clss.value()][curState.value()];
         Descriptor exe = EXE[clss.value()][curState.value()];

         if (debug) {
            char fchr = (chr == in.END)? ' ': (char)(chr & 0x00FF);
            static const string str1 = "lex    delta(\'";
            static const string str2 = "\',       ";
            string debug = str1 + fchr +  str2 + curState.geteName() + ") => <"
                         + exe.geteName() + ", " + state.geteName() 
                         + ">\n";
            out.print(debug);
            out.flush();
         }
         switch (exe.value()) {
            case 0:                                         //!< STP: endo of file found
               break;
            case 1:                                         //!< NUL: do nothing
               break;
            case 2:                                         //!< END: generate an End of File Token
               token.token(SlipRead::END);
               continue;
            case 3:                                         //!< DEC: decimal number token
               token.token(SlipRead::INTEGER, new SlipDatum((LONG)(sign*value)));
               sign = 1;
               continue;
            case 4: {                                       //!< ESC: escape sequence processing
               chr = escapeSequenceProcessing(str);
               continue; }
            case 5:                                         //!< BIN: process a decimal number
               value = value * 10 + (chr - '0');
               break;
            case 6:                                         //!< CHR: generate a character Token
               token.token(SlipRead::CHR, new SlipDatum((CHAR)str.at(1)));
               break;
            case 7:                                         //!< CTT:generate a character Token
               token.token(SlipRead::CHR, new SlipDatum((CHAR)str.at(1)));
               continue;
            case 8:  {                                      //!< DBL: generate a double    Token
               try {
                  double dbl = (double)stod(str);
                  token.token(SlipRead::DOUBL, new SlipDatum((DOUBLE)dbl));
               } catch (invalid_argument const& ex) {
                  cout << "invalid argument " << "\"" << str << "\"" << ex.what() << endl;
                  token.token(SlipRead::DOUBL, new SlipDatum((DOUBLE)0.0));
               }
               continue; }
            case 9: {                                       //!< HEX: process a hexadecimal number
               int hexvalue = ((chr - 'A') < 0)? (chr - '0')
                                               : ((chr | 0x20) - 'a' + 10);
               value = value * 16 + hexvalue;
               break; }
            case 10: {                                      //!< OCT: generate an  octal       Token
               value = value * 8 + (chr - '0');
               break; }
            case 11:                                        //!< SGN: sign (+|-) processing
               if (chr == '-') sign = -1;
               break;
            case 12:  {                                     //!< STR: generate a string Token
               int length = str.length() - 1;
               string sub = str.substr(0, length);          // substring("true" or "false")
               if (sub == "false") {
                  token.token(SlipRead::BOOLEAN, new SlipDatum((bool)false));
               } else if (sub == "true") {
                  token.token(SlipRead::BOOLEAN, new SlipDatum((bool)true));
               } else if (length == 1){
                  token.token(SlipRead::CHR, new SlipDatum((CHAR)str.at(0)));
               } else {
                  token.token(SlipRead::STR, new SlipDatum(sub));
               }
               continue; }
            case 13:                                        //!< STT: generate a   string      Token
               token.token(SlipRead::STR, new SlipDatum(str.substr(1, str.length()-2)));
               break;
            case 14:                                        //!< TOK: return delimiter    Token
               token.token(clss.type(), new SlipDatum((CHAR)str.at(0)));
               break;
            case 15:                                        //!< UIK: unsigned char token, keep chr
               token.token(SlipRead::CHR, new SlipDatum((UCHAR)str.at(1)));
               continue;
            case 16:                                        //!< UCH: unsigned char token, get next chr
               token.token(SlipRead::CHR, new SlipDatum((UCHAR)str.at(1)));
               break;
            case 17:                                        //!< UIK: unsigned int token, keep chr
               token.token(SlipRead::CHR, new SlipDatum((CHAR)value));
               continue;
            case 18:                                        //!< UIN: unsigned int token, get next chr
               token.token(SlipRead::INTEGER, new SlipDatum((ULONG)value));
               break;

         // diagnostic statements
            case 19:                                        //!< TOO: too many +|- signs
               postError(__FILE__, __LINE__, E5001, str);
               break;
            case 20:                                        //!< EXP: too many +|- signs in exponents
               postError(__FILE__, __LINE__, E5002, str);
               break;
            case 21:                                        //!< NUM: Illegal number - converting to string
               postError(__FILE__, __LINE__, E5003, str);
               break;
            default:
               break;
         };
         chr = in.getChar();                                //!< get next character
      } while (state.value() != s00.value());

      return token;
   }; // SlipRead::SlipToken SlipLexer::lexer()

   /**
     * Escape sequence processing.
     *
     * Escape sequence processing yields a single unicode character.
     * This character can be one of the special characters {\b, \f,
     * \n, \r, \t}, a unicode character, '\uxxxx', a hexadecimal or
     * an octal number.
     *
     * Processing uses an FSM, similar to the lexer FSM, to determine
     * what to return.
     *
     * Using a separate function to process escape sequences simplifies
     * the lexer FSM.
     *
     * On exit, the next character to be processed is input.
     *
     * @param[in] lexer recognized string.
     */
   char SlipLexer::escapeSequenceProcessing(StringBuilder str) {

   /**
    * Finite State Machine States.
    *
    * The values represent an index into the FSM tables, the GOTO and the
    * EXE tables, where <code>GOTO[alphabetic class][state]</code> represents
    * the transition arc to the next state in the FSM, and
    * <code>EXE[alphabetic class][state]</code> represents the function to be
    * performed at the transition.
    */
                                                            //   states
   static Descriptor s00= { 0, "s00:start "};               //!< start
   static Descriptor s01= { 1, "s01:zero  "};               //!< zero found, could be octal/hex
   static Descriptor s02= { 2, "s02:oct   "};               //!< octal number
   static Descriptor s03= { 3, "s03:dec   "};               //!< decimal number
   static Descriptor s04= { 4, "s04:hex   "};               //!< hexadecimal number

   /**
    * GOTO table shows Finite State Machine transfer arcs.
    *
    * The global state Descriptor values are used.
    *
    */
      static const Descriptor GOTO[][6] = {
      //             STATES
      //             0   OCT  DEC  HEX
      //events s00  s01  s02  s03  s04
      /*EOF*/ {s00, s00, s00, s00, s00}
      /*spc*/,{s00, s00, s00, s00, s00}
      /*zro*/,{s01, s02, s02, s03, s04}
      /*oct*/,{s03, s02, s02, s03, s04}
      /*dec*/,{s03, s00, s00, s03, s04}
      /*hex*/,{s00, s00, s00, s00, s04}
      /*alp*/,{s00, s00, s00, s00, s00}
      /*lpr*/,{s00, s00, s00, s00, s00}
      /*rpr*/,{s00, s00, s00, s00, s00}
      /*lcr*/,{s00, s00, s00, s00, s00}
      /*rcr*/,{s00, s00, s00, s00, s00}
      /*lng*/,{s00, s00, s00, s00, s00}
      /*rng*/,{s00, s00, s00, s00, s00}
      /*cln*/,{s00, s00, s00, s00, s00}
      /*squ*/,{s00, s00, s00, s00, s00}
      /*dqu*/,{s00, s00, s00, s00, s00}
      /*sls*/,{s00, s00, s00, s00, s00}
      /*esc*/,{s00, s00, s00, s00, s00}
      /*ast*/,{s00, s00, s00, s00, s00}
      /*plu*/,{s00, s00, s00, s00, s00}
      /*min*/,{s00, s00, s00, s00, s00}
      /*per*/,{s00, s00, s00, s00, s00}
      /*x  */,{s00, s04, s00, s00, s00}
      /*d  */,{s00, s00, s00, s00, s04}
      /*e  */,{s00, s00, s00, s00, s04}
              }; // Descriptor GOTO[][6]

      /**
       * Arc functions.
       *
       * At each FSM transition a function is executed. The values given stand for
       * the switch statement value whUCH selects a case branch statement. This
       * brach statement performs the indicated operation.
       */
                                                            //   FSM function definition
      static Descriptor nul= { 0, "nul   "};                //!< do nothing
      static Descriptor end= { 1, "stop  "};                //!< terminate processing
      static Descriptor OCT= { 2, "octal "};                //!< terminate octal processing
      static Descriptor DEC= { 3, "dec   "};                //!< terminate decimal processing
      static Descriptor HEX= { 4, "hex   "};                //!< terminate hexadecimal processing
      static Descriptor RMV= { 5, "remove"};                //!< remove "\0x" from input

      static const Descriptor EXE[][6] = {
      //             STATES
      //             0   OCT  DEC  HEX
      //events end  s01  s02  s03  s04
      /*EOF*/ {end, end, end, end, end}
      /*spc*/,{end, end, end, end, end}
      /*zro*/,{nul, nul, OCT, DEC, HEX}
      /*oct*/,{nul, OCT, OCT, DEC, HEX}
      /*dec*/,{nul, DEC, end, DEC, HEX}
      /*hex*/,{end, end, end, end, HEX}
      /*alp*/,{end, end, end, end, end}
      /*lpr*/,{end, end, end, end, end}
      /*rpr*/,{end, end, end, end, end}
      /*lcr*/,{end, end, end, end, end}
      /*rcr*/,{end, end, end, end, end}
      /*lng*/,{end, end, end, end, end}
      /*rng*/,{end, end, end, end, end}
      /*cln*/,{end, end, end, end, end}
      /*squ*/,{end, end, end, end, end}
      /*dqu*/,{end, end, end, end, end}
      /*sls*/,{end, end, end, end, end}
      /*esc*/,{end, end, end, end, end}
      /*ast*/,{end, end, end, end, end}
      /*plu*/,{end, end, end, end, end}
      /*min*/,{end, end, end, end, end}
      /*per*/,{end, end, end, end, end}
      /*x  */,{end, nul, end, end, end}
      /*d  */,{end, end, end, end, HEX}
      /*e  */,{end, end, end, end, HEX}
              }; // Descriptor EXE[][6]

      Descriptor state  = s00;                              //!< current FSM state
      int  value  = 0;                                      //!< excaped value to be inserted
      char c_string[2] = {'\0', '\0'};

   /*********** escapeSequenceProcessing code **********/

      chr = in.getChar();                                   //!< Escape (\\) recognized, get next character
      if (chr > 0x00FF) return chr;                         //!< unicode character

      if ((chr | 0x0020) == 'u') {                          //!< unicode character must be hexadecimal
         state = s04;
         chr = in.getChar();
      } else if (map[chr].value() == alp.value()) {         //!< Alphabetic character
         if      (chr == 'b') chr = '\b';
         else if (chr == 'f') chr = '\f';
         else if (chr == 'n') chr = '\n';
         else if (chr == 'r') chr = '\r';
         else if (chr == 't') chr = '\t';
         str.append(&(c_string[0] = chr));
         return in.getChar();
      } else if (map[chr].value() == esc.value()) {
         str.append(&(c_string[0] = chr));
         return in.getChar();
      }

      do {
         Descriptor clss = (chr > 0x00FF)? alp: map[chr];   //!< character class
         Descriptor curState = state;

         state          = GOTO[clss.value()][curState.value()];
         Descriptor exe = EXE[clss.value()][curState.value()];

         if (debug) {
            char fchr = (chr == in.END)? ' ': chr;
            static const string str1 = "escape delta(\'";
            static const string str2 = "\',       ";
            string debug = str1 + fchr +  str2 + curState.geteName() + ") => <"
                         + exe.geteName() + ", " + state.geteName() + ">\n" ;
            out.print(debug);
            out.flush();
         }
         try {
            switch(exe.value()) {
               case 0:                                      //!< nul: do nothing
                  break;
               case 1:                                      //!< stop
                  continue;
               case 2:                                      //!< octal
                  value = (value * 8) + (chr - '0');
                  break;
               case 3:                                      //!< decimal
                  value = (value * 10) + (chr - '0');
                  break;
               case 4:                                      //!< hexadecimal
                  int hexvalue = ((chr - 'A') < 0)? (chr - '0')
                                                  : ((chr | 0x20) - 'a' + 10);
                  value = (value * 16) + hexvalue;
                  break;
            };
         } catch(exception e) {
            state = s00;
         }
         chr = in.getChar();
      } while (state.value() != s00.value());

      str.append(&(c_string[0] = (char)value));
      return chr;
   }; // char SlipLexer::escapeSequenceProcessing(SlipInputInterface in)

}; // namespace slip