nxstoken.h

//  Copyright (C) 1999-2003 Paul O. Lewis
//
//  This file is part of NCL (Nexus Class Library) version 2.0.
//
//  NCL is free software; you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation; either version 2 of the License, or
//  (at your option) any later version.
//
//  NCL is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
//  GNU General Public License for more details.
//
//  You should have received a copy of the GNU General Public License
//  along with NCL; if not, write to the Free Software Foundation, Inc.,
//  59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//

#ifndef NCL_NXSTOKEN_H
#define NCL_NXSTOKEN_H

#include "ncl/nxsexception.h"
class NxsToken;

class NxsX_UnexpectedEOF: public NxsException
    {
    public:
        NxsX_UnexpectedEOF(NxsToken &);
    };


class NxsComment
    {
    public:
        NxsComment(const std::string & s, long lineNumber, long colNumber)
            :body(s),
            line(lineNumber),
            col(colNumber)
            {}
        long        GetLineNumber() const
            {
            return line;
            }
        long        GetColumnNumber() const
            {
            return col;
            }
        const std::string & GetText() const
            {
            return body;
            }
        void WriteAsNexus(std::ostream &out) const
            {
            out << '[' << body << ']';
            }
    private:
        std::string body;
        long line;
        long col;
    };


class NxsTokenPosInfo
    {
    public:
        NxsTokenPosInfo()
            :pos(0),
            line(-1),
            col(-1)
            {}
        NxsTokenPosInfo(file_pos position, long lineno, long columnno)
            :pos(position),
            line(lineno),
            col(columnno)
            {}
        NxsTokenPosInfo(const NxsToken &);

        file_pos    GetFilePosition() const
            {
            return pos;
            }

        long        GetLineNumber() const
            {
            return line;
            }

        long        GetColumnNumber() const
            {
            return col;
            }


        file_pos    pos;    /* current file position */
        long        line;   /* current line in file */
        long        col;    /* column of current line */
    };


class NxsSimpleCommandStrings
    {
    public:
        typedef std::vector<std::string> VecString;
        typedef std::list<VecString> MatString;
        typedef std::pair<NxsTokenPosInfo, std::string> SingleValFromFile;
        typedef std::pair<NxsTokenPosInfo, VecString > MultiValFromFile;
        typedef std::pair<NxsTokenPosInfo, MatString > MatFromFile;
        typedef std::map<std::string, SingleValFromFile> StringToValFromFile;
        typedef std::map<std::string,  MultiValFromFile> StringToMultiValFromFile;
        typedef std::map<std::string,  MatFromFile> StringToMatFromFile;

        // Looks for k in opts and multiOpts. Returns all of the values
        //  for the command option (will be an empty vector of strings if the option was not found).
        // Case-sensitive!
        // If an option is in multiOpts and opts, then only the value from opts will be returned!
        MultiValFromFile GetOptValue(const std::string &k) const
            {
            MultiValFromFile mvff;
            StringToValFromFile::const_iterator s = this->opts.find(k);
            if (s != this->opts.end())
                {
                const SingleValFromFile & v(s->second);
                mvff.first = v.first;
                mvff.second.push_back(v.second);
                }
            else
                {
                StringToMultiValFromFile::const_iterator m = this->multiOpts.find(k);
                if (m != this->multiOpts.end())
                    {
                    const MultiValFromFile & mv(m->second);
                    mvff.first = mv.first;
                    mvff.second  = mv.second;
                    }
                }
            return mvff;
            }

        MatFromFile GetMatOptValue(const std::string & k) const
            {
            StringToMatFromFile::const_iterator mIt = this->matOpts.find(k);
            if (mIt ==  this->matOpts.end())
                return MatFromFile();
            return mIt->second;
            }

        bool HasKey(const std::string k) const
            {
            if (this->opts.find(k) !=  this->opts.end())
                return true;
            return ((this->multiOpts.find(k) !=  this->multiOpts.end()) || (this->matOpts.find(k) !=  this->matOpts.end()));
            }

        std::string cmdName;
        NxsTokenPosInfo cmdPos;
        StringToValFromFile opts;
        StringToMultiValFromFile multiOpts;
        StringToMatFromFile matOpts;
    };

class ProcessedNxsToken
    {
    public:
        static void IncrementNotLast(std::vector<ProcessedNxsToken>::const_iterator & it,
                                     const std::vector<ProcessedNxsToken>::const_iterator &endIt,
                                     const char * context);
        static NxsSimpleCommandStrings ParseSimpleCmd(const std::vector<ProcessedNxsToken> &, bool convertToLower);


        ProcessedNxsToken(const NxsToken &t);

        ProcessedNxsToken(std::string &s)
            :token(s)
            {}

        ProcessedNxsToken(std::string &s, file_pos position,long lineno, long columnno)
            :token(s),
            posInfo(position, lineno, columnno)
            {}

        std::string GetToken() const
            {
            return token;
            }

        const std::vector<NxsComment> & GetEmbeddedComments() const
            {
            return embeddedComments;
            }

        NxsTokenPosInfo     GetFilePosInfo() const
            {
            return posInfo;
            }
        const NxsTokenPosInfo & GetFilePosInfoConstRef() const
            {
            return posInfo;
            }

        file_pos    GetFilePosition() const
            {
            return posInfo.GetFilePosition();
            }

        long        GetLineNumber() const
            {
            return posInfo.GetLineNumber();
            }

        long        GetColumnNumber() const
            {
            return posInfo.GetColumnNumber();
            }

        bool        Equals(const char *c) const
            {
            return NxsString::case_insensitive_equals(token.c_str(), c);
            }
        bool        EqualsCaseSensitive(const char *c) const
            {
            return (strcmp(token.c_str(), c) == 0);
            }

        void        SetEmbeddedComments(const std::vector<NxsComment> & c)
            {
            embeddedComments = c;
            }

        void WriteAsNexus(std::ostream &out) const
            {
            for(std::vector<NxsComment>::const_iterator cIt = embeddedComments.begin(); cIt != embeddedComments.end(); ++cIt)
                cIt->WriteAsNexus(out);
            out << NxsString::GetEscaped(token);
            }
    private:
        std::string token;
        NxsTokenPosInfo posInfo;
        std::vector<NxsComment> embeddedComments; /* comments that were processed in the same GetToken operation that created this token. */
    };

typedef std::vector<ProcessedNxsToken> ProcessedNxsCommand;
bool WriteCommandAsNexus(std::ostream &, const ProcessedNxsCommand &);


class NxsToken
    {
    public:
        static std::string  EscapeString(const std::string &);
        static bool         NeedsQuotes(const std::string &);
        static std::string  GetQuoted(const std::string &);
        static void         DemandEndSemicolon(NxsToken &token, NxsString & errormsg, const char *contextString);
        static unsigned     DemandPositiveInt(NxsToken &token, NxsString & errormsg, const char *contextString);
        static std::map<std::string, std::string> ParseAsSimpleKeyValuePairs(const ProcessedNxsCommand & tv, const char *cmdName);

        enum NxsTokenFlags  /* For use with the variable labileFlags */
            {
            saveCommandComments     = 0x0001,   /* if set, command comments of the form [&X] are not ignored but are instead saved as regular tokens (without the square brackets, however) */
            parentheticalToken      = 0x0002,   /* if set, and if next character encountered is a left parenthesis, token will include everything up to the matching right parenthesis */
            curlyBracketedToken     = 0x0004,   /* if set, and if next character encountered is a left curly bracket, token will include everything up to the matching right curly bracket */
            doubleQuotedToken       = 0x0008,   /* if set, grabs entire phrase surrounded by double quotes */
            singleCharacterToken    = 0x0010,   /* if set, next non-whitespace character returned as token */
            newlineIsToken          = 0x0020,   /* if set, newline character treated as a token and atEOL set if newline encountered */
            tildeIsPunctuation      = 0x0040,   /* if set, tilde character treated as punctuation and returned as a separate token */
            useSpecialPunctuation   = 0x0080,   /* if set, character specified by the data member special is treated as punctuation and returned as a separate token */
            hyphenNotPunctuation    = 0x0100,   /* if set, the hyphen character is not treated as punctutation (it is normally returned as a separate token) */
            preserveUnderscores     = 0x0200,   /* if set, underscore characters inside tokens are not converted to blank spaces (normally, all underscores are automatically converted to blanks) */
            ignorePunctuation       = 0x0400    /* if set, the normal punctuation symbols are treated the same as any other darkspace characters */
            };

        NxsString       errormsg;

                        NxsToken(std::istream &i);
        virtual         ~NxsToken();

        bool            AtEOF();
        bool            AtEOL();
        bool            Abbreviation(NxsString s);
        bool            Begins(NxsString s, bool respect_case = false);
        void            BlanksToUnderscores();
        bool            Equals(NxsString s, bool respect_case = false) const;
        bool        EqualsCaseSensitive(const char *c) const
            {
            return (strcmp(token.c_str(), c) == 0);
            }

        long            GetFileColumn() const;
        file_pos        GetFilePosition() const;
        long            GetFileLine() const;
        void            GetNextToken();
        NxsString       GetToken(bool respect_case = true);
        const char      *GetTokenAsCStr(bool respect_case = true);
        const NxsString &GetTokenReference() const;
        int             GetTokenLength() const;
        bool            IsPlusMinusToken();
        bool            IsPunctuationToken();
        bool            IsWhitespaceToken();
        bool            IsPlusMinusToken(const std::string & t);
        bool            IsPunctuationToken(const std::string & t);
        bool            IsWhitespaceToken(const std::string & t);
        std::map<std::string, std::string> ProcessAsSimpleKeyValuePairs(const char *cmdName);
        void            ProcessAsCommand(ProcessedNxsCommand *tokenVec);
        void            ReplaceToken(const NxsString s);
        void            ResetToken();
        void            SetSpecialPunctuationCharacter(char c);
        void            SetLabileFlagBit(int bit);
        bool            StoppedOn(char ch);
        void            StripWhitespace();
        void            ToUpper();
        void            Write(std::ostream &out);
        void            Writeln(std::ostream &out);

        virtual void    OutputComment(const NxsString &msg);

        void            SetEOFAllowed(bool e)
            {
            eofAllowed = e;
            }
        bool            GetEOFAllowed() const
            {
            return eofAllowed;
            }
        void            SetBlockName(const char *);
        std::string     GetBlockName();
        const std::vector<NxsComment> & GetEmbeddedComments() const
            {
            return embeddedComments;
            }
        char            PeekAtNextChar() const;
    protected:

        void            AppendToComment(char ch);
        void            AppendToToken(char ch);
        bool            GetComment();
        void            GetCurlyBracketedToken();
        void            GetDoubleQuotedToken();
        void            GetQuoted();
        void            GetParentheticalToken();
        bool            IsPunctuation(char ch);
        bool            IsWhitespace(char ch);

    private:
        void AdvanceToNextCharInStream();
        char            GetNextChar();
        //char ReadNextChar();

        std::istream    &inputStream;       /* reference to input stream from which tokens will be read */
        signed char     nextCharInStream;
        file_pos        posOffBy;           /* offset of the file pos (according to the stream) and the tokenizer (which is usually a character or two behind, due to saved chars */
        file_pos        usualPosOffBy;      /* default of posOffBy.  Usually this is -1, but it can be positive if a tokenizer is created from a substring of the file */
        long            fileLine;           /* current file line */
        long            fileColumn;         /* current column in current line (refers to column immediately following token just read) */
        NxsString       token;              /* the character buffer used to store the current token */
        NxsString       comment;            /* temporary buffer used to store output comments while they are being built */
        bool            eofAllowed;
        signed char     saved;              /* either '\0' or is last character read from input stream */
        bool            atEOF;              /* true if end of file has been encountered */
        bool            atEOL;              /* true if newline encountered while newlineIsToken labile flag set */
        char            special;            /* ad hoc punctuation character; default value is '\0' */
        int             labileFlags;        /* storage for flags in the NxsTokenFlags enum */
        char            whitespace[4];      /* stores the 3 whitespace characters: blank space, tab and newline */
        std::string     currBlock;
        std::vector<NxsComment>     embeddedComments;
    };

typedef NxsToken NexusToken;


inline ProcessedNxsToken::ProcessedNxsToken(const NxsToken &t)
    :token(t.GetTokenReference()),
    posInfo(t)
    {}

inline NxsTokenPosInfo::NxsTokenPosInfo(const NxsToken &t)
    :pos(t.GetFilePosition()),
    line(t.GetFileLine()),
    col(t.GetFileColumn())
    {}

inline void NxsToken::SetBlockName(const char *c)
    {
    if (c == 0L)
        currBlock.clear();
    else
        currBlock.assign(c);
    }

inline std::string NxsToken::GetBlockName()
    {
    return currBlock;
    }

inline std::string NxsToken::EscapeString(const std::string &s)
    {
    return NxsString::GetEscaped(s);
    }

inline const NxsString &NxsToken::GetTokenReference() const
    {
    return token;
    }

inline void NxsToken::OutputComment(
  const NxsString &)    /* the contents of the printable comment discovered in the NEXUS data file */
    {
    }

inline void NxsToken::AppendToComment(
  char ch)  /* character to be appended to comment */
    {
    comment += ch;
    }

inline void NxsToken::AppendToToken(
  char ch)  /* character to be appended to token */
    {
    token.push_back(ch);
    }

inline bool NxsToken::IsPunctuation(
  char ch)  /* the character in question */
    {
    // PAUP 4.0b10
    //  o allows ]`<> inside taxon names
    //  o allows `<> inside taxset names
    //
    if (NxsString::IsNexusPunctuation(ch))
        {
        if  (labileFlags & hyphenNotPunctuation)
#           if defined(NCL_VERSION_2_STYLE_HYPHEN) && NCL_VERSION_2_STYLE_HYPHEN
                return (ch != '-');
#           else
                return (ch != '-'  && ch != '+');
#           endif
        return true;
        }
    if (labileFlags & tildeIsPunctuation  && ch == '~')
        return true;
    return (labileFlags & useSpecialPunctuation  && ch == special);
    }

inline bool NxsToken::IsWhitespace(
  char ch)  /* the character in question */
    {
    bool ws = false;

    // If ch is found in the whitespace array, it's whitespace
    //
    if (strchr(whitespace, ch) != NULL)
        ws = true;

    // Unless of course ch is the newline character and we're currently
    // treating newlines as darkspace!
    //
    if (labileFlags & newlineIsToken && ch == '\n')
        ws = false;

    return ws;
    }

inline bool NxsToken::AtEOF()
    {
    return atEOF;
    }

inline bool NxsToken::AtEOL()
    {
    return atEOL;
    }

inline void NxsToken::BlanksToUnderscores()
    {
    token.BlanksToUnderscores();
    }

inline long  NxsToken::GetFileColumn() const
    {
    return fileColumn;
    }

inline file_pos  NxsToken::GetFilePosition() const
    {
    return inputStream.rdbuf()->pubseekoff(0,std::ios::cur, std::ios::in) + posOffBy;
    }

inline long  NxsToken::GetFileLine() const
    {
    return fileLine;
    }

inline NxsString NxsToken::GetToken(
  bool respect_case)    /* determines whether token is converted to upper case before being returned */
    {
    if (!respect_case)
        ToUpper();

    return token;
    }

inline const char *NxsToken::GetTokenAsCStr(
  bool respect_case)    /* determines whether token is converted to upper case before being returned */
    {
    if (!respect_case)
        ToUpper();

    return token.c_str();
    }

inline int NxsToken::GetTokenLength() const
    {
    return (int)token.size();
    }

inline bool NxsToken::IsPlusMinusToken()
    {
    return IsPlusMinusToken(token);
    }

inline bool NxsToken::IsPlusMinusToken(const std::string &t)
    {
    return (t.size() == 1 && ( t[0] == '+' || t[0] == '-') );
    }

inline bool NxsToken::IsPunctuationToken()
    {
    return IsPunctuationToken(token);
    }

inline bool NxsToken::IsPunctuationToken(const std::string &t)
    {
    return (t.size() == 1 && IsPunctuation(t[0]));
    }


inline bool NxsToken::IsWhitespaceToken()
    {
    return IsWhitespaceToken(token);
    }

inline bool NxsToken::IsWhitespaceToken(const std::string &t)
    {
    return (t.size() == 1 && IsWhitespace( t[0]));
    }

inline void NxsToken::ReplaceToken(
  const NxsString s)    /* NxsString to replace current token NxsString */
    {
    token = s;
    }

inline void NxsToken::ResetToken()
    {
    token.clear();
    embeddedComments.clear();
    }

inline void NxsToken::SetSpecialPunctuationCharacter(
  char c)   /* the character to which `special' is set */
    {
    special = c;
    }

inline void NxsToken::SetLabileFlagBit(
  int bit)  /* the bit (see NxsTokenFlags enum) to set in `labileFlags' */
    {
    labileFlags |= bit;
    }

inline bool NxsToken::StoppedOn(
  char ch)  /* the character to compare with saved character */
    {
    if (saved == ch)
        return true;
    else
        return false;
    }
inline char NxsToken::PeekAtNextChar() const
    {
    return nextCharInStream;
    }
inline void NxsToken::Write(
  std::ostream &out)    /* the output stream to which to write token NxsString */
    {
    out << token;
    }

inline void NxsToken::Writeln(
  std::ostream &out)    /* the output stream to which to write `token' */
    {
    out << token << std::endl;
    }

inline std::map<std::string, std::string> NxsToken::ProcessAsSimpleKeyValuePairs(const char *cmdName)
    {
    ProcessedNxsCommand tokenVec;
    ProcessAsCommand(&tokenVec);
    return ParseAsSimpleKeyValuePairs(tokenVec, cmdName);
    }

inline bool NxsToken::Equals(
  NxsString s, /* the string for comparison to the string currently stored in this token */
  bool respect_case) const  /* if true, comparison will be case-sensitive */
    {
    if (respect_case)
        return (strcmp(token.c_str(), s.c_str()) == 0);
    return NxsString::case_insensitive_equals(token.c_str(), s.c_str());
    }

#endif

---