nxstoken.cpp

//  Copyright (C) 1999-2003 Paul O. Lewis
//
//  This file is part of NCL (Nexus Class Library) version 2.0.
//
//  NCL is free software; you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation; either version 2 of the License, or
//  (at your option) any later version.
//
//  NCL is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
//  GNU General Public License for more details.
//
//  You should have received a copy of the GNU General Public License
//  along with NCL; if not, write to the Free Software Foundation, Inc.,
//  59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
#include <cstdlib>
#include <cassert>
#include "ncl/nxstoken.h"

using namespace std;

#define NEW_NXS_TOKEN_READ_CHAR




NxsSimpleCommandStrings ProcessedNxsToken::ParseSimpleCmd(
  const std::vector<ProcessedNxsToken> &pnc,
    bool convertToLower)
{
    NxsSimpleCommandStrings nscs;
    if (pnc.empty())
        return nscs;


    std::vector<ProcessedNxsToken>::const_iterator wordIt = pnc.begin();

    nscs.cmdName = wordIt->GetToken();
    if (convertToLower)
        NxsString::to_lower(nscs.cmdName);
    nscs.cmdPos = wordIt->GetFilePosInfoConstRef();
    ++wordIt;

    std::string key;

    NxsString errorMsg;
    NxsTokenPosInfo keyPos = nscs.cmdPos;
    bool eqRead = false;
    for (; wordIt != pnc.end(); ++wordIt)
        {
        std::string w = wordIt->GetToken();
        if (convertToLower)
            NxsString::to_lower(w);
        if (key.empty())
            {
            key = w;
            if (nscs.HasKey(key))
                {
                errorMsg << "Command option (" << key << ") repeated in the " << nscs.cmdName << " command.";
                throw NxsException(errorMsg, wordIt->GetFilePosInfoConstRef());
                }
            keyPos = wordIt->GetFilePosInfoConstRef();
            }
        else if (!eqRead)
            {
            if (w != "=")
                {
                errorMsg << "Expecting an = after the  " << key << " command option of the  " << nscs.cmdName << " command.";
                throw NxsException(errorMsg, wordIt->GetFilePosInfoConstRef());
                }
            eqRead = true;
            }
        else {
            if (w == "(")
                {
                ++wordIt;
                w = wordIt->GetToken();
                std::vector<std::string> vals;
                NxsSimpleCommandStrings::MatString mat;
                if (w == "(")
                    {
                    while (w != ")")
                        {
                        if (w != "(")
                            {
                            errorMsg << "Expecting a ( to begin another row of values in the " << key << " command option of the  " << nscs.cmdName << " command.";
                            throw NxsException(errorMsg, keyPos);
                            }

                        ++wordIt;
                        w = wordIt->GetToken();
                        while (wordIt != pnc.end())
                            {
                            w = wordIt->GetToken();
                            if (convertToLower)
                                NxsString::to_lower(w);
                            if (w == ")")
                                break;
                            vals.push_back(w);
                            ++wordIt;
                            }
                        if (wordIt == pnc.end())
                            {
                            errorMsg << "Expecting a ) to end the list of values for the " << key << " command option of the  " << nscs.cmdName << " command.";
                            throw NxsException(errorMsg, keyPos);
                            }
                        ++wordIt;
                        mat.push_back(vals);
                        vals.clear();
                        w = wordIt->GetToken();
                        if (wordIt == pnc.end())
                            {
                            errorMsg << "Expecting a ) to end the list of values for the " << key << " command option of the  " << nscs.cmdName << " command.";
                            throw NxsException(errorMsg, keyPos);
                            }
                        }
                    nscs.matOpts[key] = NxsSimpleCommandStrings::MatFromFile(wordIt->GetFilePosInfoConstRef(), mat);
                    }
                else
                    {
                    while (wordIt != pnc.end())
                        {
                        w = wordIt->GetToken();
                        if (convertToLower)
                            NxsString::to_lower(w);
                        if (w == ")")
                            break;
                        vals.push_back(w);
                        ++wordIt;
                        }
                    if (wordIt == pnc.end())
                        {
                        errorMsg << "Expecting a ) to end the list of values for the " << key << " command option of the  " << nscs.cmdName << " command.";
                        throw NxsException(errorMsg, keyPos);
                        }
                    nscs.multiOpts[key] = NxsSimpleCommandStrings::MultiValFromFile(wordIt->GetFilePosInfoConstRef(), vals);
                    }
                }
            else
                {
                std::string val = w;
                nscs.opts[key] = NxsSimpleCommandStrings::SingleValFromFile( wordIt->GetFilePosInfoConstRef(), val);
                }
            eqRead = false;
            key.clear();
            }
        }
    if (eqRead)
        {
        errorMsg << "Expecting a value after the = sign in the  " << key << " command option of the  " << nscs.cmdName << " command.";
        throw NxsException(errorMsg, keyPos);
        }
    if (!key.empty())
        {
        errorMsg << "Expecting an = after the  " << key << " command option of the  " << nscs.cmdName << " command.";
        throw NxsException(errorMsg, keyPos);
        }
    return nscs;
}




NxsX_UnexpectedEOF::NxsX_UnexpectedEOF(NxsToken &token)
    :NxsException("Unexpected end-of-file", token)
    {
    std::string t = token.GetBlockName();
    NxsString::to_upper(t);
    if (!t.empty())
        msg << " while reading " << t << " block.";
    }

bool WriteCommandAsNexus(std::ostream & out, const ProcessedNxsCommand &c)
    {
    if (c.empty())
        return false;
    out << "   "; /* command indentation  - 1 space*/
    for(ProcessedNxsCommand::const_iterator cIt = c.begin(); cIt != c.end(); ++cIt)
        {
        out << ' ';
        cIt->WriteAsNexus(out);
        }
    out << ";";
    return true;
    }



void ProcessedNxsToken::IncrementNotLast(std::vector<ProcessedNxsToken>::const_iterator & tokIt, const std::vector<ProcessedNxsToken>::const_iterator &endIt, const char * contextString)
    {
    ++tokIt;
    if (tokIt == endIt)
        {
        NxsString errormsg = "Unexpected ; ";
        if (contextString)
            errormsg.append(contextString);
        --tokIt;
        throw NxsException(errormsg, *tokIt);
        }
    }
inline void NxsToken::AdvanceToNextCharInStream()
    {
    if (nextCharInStream == EOF)
        return;
    nextCharInStream  = (signed char) (inputStream.rdbuf())->sbumpc();
    posOffBy = -1;
    if (nextCharInStream == 13 || nextCharInStream == 10)
        {
        if(nextCharInStream == 13)
            {
            if ((inputStream.rdbuf())->sgetc() == 10)   //peeks at the next char
                {
                (inputStream.rdbuf())->sbumpc();
                posOffBy = -2;
                }
            }
        nextCharInStream = '\n';
        }
    }


#if defined(NEW_NXS_TOKEN_READ_CHAR)

inline char NxsToken::GetNextChar()
    {
    //
    //  Why this was changed:  calls to tellg seem slow and unnecessary - we're storing filepos in terms of the
    //  number of times we call sbumpc().
    //  if we go back to getting the filepos via in.tellg(), remember to call it
    //  twice after both sgetc() calls in the case of the \13\10 endline

    signed char ch = nextCharInStream;
    AdvanceToNextCharInStream();
    if(ch == EOF)
        {
        atEOF = true;
        if (eofAllowed)
            return '\0';
        throw NxsX_UnexpectedEOF(*this);
        }
    if(ch == '\n')
        {
        fileLine++;
        fileColumn = 1L;
        atEOL = true;
        return '\n';
        }
    if (ch == '\t')
        fileColumn += 4 - ((fileColumn - 1)%4); //@assumes that tab will be 4 in the editor we use
    else
        fileColumn++;
    atEOL = false;
    return ch;
    }

#else //    if  !defined(NEW_NXS_TOKEN_READ_CHAR)

inline char NxsToken::GetNextChar()
    {
    int ch = inputStream.get();
    int failed = inputStream.bad();
    if (failed)
        {
        errormsg = "Unknown error reading data file (check to make sure file exists)";
        throw NxsException(errormsg);
        }

    if (ch == 13 || ch == 10)
        {
        fileLine++;
        fileColumn = 1L;

        if (ch == 13 && (int)inputStream.peek() == 10)
            ch = inputStream.get();

        atEOL = 1;
        }
    else if (ch == EOF)
        atEOF = 1;
    else
        {
        fileColumn++;
        atEOL = 0;
        }

#   if defined(__DECCXX)
        filepos = 0L;
#   else
        file_pos filepos = inputStream.tellg();
#   endif

    if (atEOF)
        return '\0';
    else if (atEOL)
        return '\n';
    else
        return (char)ch;
    }
#endif

std::map<std::string, std::string> NxsToken::ParseAsSimpleKeyValuePairs(const ProcessedNxsCommand & tv, const char *cmdName)
    {
    std::map<std::string, std::string> kv;
    std::string key;
    ProcessedNxsCommand::const_iterator tvIt = tv.begin();
    ProcessedNxsCommand::const_iterator prevIt;
    ProcessedNxsCommand::const_iterator endIt = tv.end();
    while (tvIt != endIt)
        {
        key = tvIt->GetToken().c_str();
        prevIt = tvIt++;
        if (tvIt == endIt || tvIt->GetToken() != "=")
            {
            NxsString m("Expecting = after ");
            m += key.c_str();
            m += " in ";
            m += cmdName;
            m += " command.";
            if (tvIt == endIt)
                throw NxsException(m, prevIt->GetFilePosition(), prevIt->GetLineNumber(), prevIt->GetColumnNumber());
            else
                throw NxsException(m, tvIt->GetFilePosition(), tvIt->GetLineNumber(), tvIt->GetColumnNumber());
            }
        prevIt = tvIt++;
        if (tvIt == endIt)
            {
            NxsString m("Expecting a value after = in the  ");
            m += key.c_str();
            m += " subcommand of the in ";
            m += cmdName;
            m += " command.";
            throw NxsException(m, prevIt->GetFilePosition(), prevIt->GetLineNumber(), prevIt->GetColumnNumber());
            }
        kv[key] = tvIt->GetToken();
        tvIt++;
        }
    return kv;
    }
void NxsToken::ProcessAsCommand(ProcessedNxsCommand *tokenVec)
    {
    ;
    while (!this->Equals(";"))
        {
        if (tokenVec)
            tokenVec->push_back(ProcessedNxsToken(*this));
        this->GetNextToken();
        }
    }


std::string NxsToken::GetQuoted(const std::string &s)
    {
    std::string withQuotes;
    withQuotes.reserve(s.length() + 4);
    withQuotes.push_back('\'');
    for (NxsString::const_iterator sIt = s.begin(); sIt != s.end(); sIt++)
        {
        withQuotes.push_back(*sIt);
        if (*sIt == '\'')
            withQuotes.push_back('\'');
        }
    withQuotes.push_back('\'');
    return withQuotes;
    }

unsigned NxsToken::DemandPositiveInt(NxsToken &token, NxsString & errormsg, const char *contextString)
    {
    token.GetNextToken();
    int i = atoi(token.GetToken().c_str());
    if (i <= 0)
        {
        errormsg.assign(contextString);
        errormsg += " must be a number greater than 0. Found ";
        errormsg += token.GetToken();
        errormsg += " instead";
        throw NxsException(errormsg, token.GetFilePosition(), token.GetFileLine(), token.GetFileColumn());
        }
    return (unsigned) i;
    }


void NxsToken::DemandEndSemicolon(NxsToken &token, NxsString & errormsg, const char *contextString)
    {
    token.GetNextToken();
    if (!token.Equals(";"))
        {
        errormsg = "Expecting ';' to terminate the ";
        errormsg += contextString;
        errormsg += " command, but found ";
        errormsg += token.GetToken();
        errormsg += " instead";
        throw NxsException(errormsg, token.GetFilePosition(), token.GetFileLine(), token.GetFileColumn());
        }
    }
bool NxsToken::NeedsQuotes(const std::string &s)
    {
    for (std::string::const_iterator sIt = s.begin(); sIt != s.end(); sIt++)
        {
        const char &c = (*sIt);
        if (!isgraph(c))
            return true;
        else if (strchr("\'[(){}\"-]/\\,;:=*`+<>", c) != NULL)
            {
            // ' and [ always need quotes.  other punctuation needs quotes if it is in a word of length > 1
            if (c == '\'' || c == '[')
                return true;
            return (s.length() > 1);
            }
        }
    return false;
    }



NxsToken::NxsToken(
  istream &i)   /* the istream object to which the token is to be associated */
  : inputStream(i),
    eofAllowed(true)
    {
    posOffBy = 0;
    atEOF       = false;
    atEOL       = false;
    comment.clear();
    fileColumn  = 1L;
    fileLine    = 1L;
    labileFlags = 0;
    saved       = '\0';
    special     = '\0';

    whitespace[0]  = ' ';
    whitespace[1]  = '\t';
    whitespace[2]  = '\n';
    whitespace[3]  = '\0';
#   if defined(NEW_NXS_TOKEN_READ_CHAR)
        nextCharInStream = 'a'; //anything other than EOF will work
        AdvanceToNextCharInStream();
#   endif
    }

NxsToken::~NxsToken()
    {
    }

bool NxsToken::GetComment()
    {
    // Set comment level to 1 initially.  Every ']' encountered reduces
    // level by one, so that we know we can stop when level becomes 0.
    //
    NxsString currentComment;
    bool command = false;

    bool formerEOFAllowed = eofAllowed;
    eofAllowed = false;
    try
        {
        char ch = GetNextChar();
        // See if first character is the output comment symbol ('!')
        // or command comment symbol (&)
        //
        int printing = 0;
        if (ch == '!')
            printing = 1;
        else if (ch == '&' && (labileFlags & saveCommandComments) && token.empty())
            command = true;
        currentComment.push_back(ch);
        if (ch != ']')
            {
            int level = 1;
            for(;;)
                {
                ch = GetNextChar();
                if (ch == ']')
                    {
                    level--;
                    if (level == 0)
                        break;
                    }
                else if (ch == '[')
                    level++;
                currentComment.push_back(ch);
                }

            if (printing)
                {
                // Allow output comment to be printed or displayed in most appropriate
                // manner for target operating system
                //
                NxsString foroutput(currentComment.c_str() + 1);
                comment = foroutput;
                OutputComment(foroutput);
                }
            if (command)
                token = currentComment;
            else
                embeddedComments.push_back(NxsComment(currentComment, GetFileLine(), GetFileColumn()));
            }
        }
    catch (NxsX_UnexpectedEOF & x)
        {
        x.msg << " (end-of-file inside comment)";
        eofAllowed = formerEOFAllowed;
        throw x;
        }
    eofAllowed = formerEOFAllowed ;
    return command;
    }

void NxsToken::GetCurlyBracketedToken()
    {
    bool formerEOFAllowed = eofAllowed;
    eofAllowed = false;
    try
        {
        int level = 1;
        while(level > 0)
            {
            char ch = GetNextChar();
            if (ch == '}')
                level--;
            else if (ch == '{')
                level++;
            AppendToToken(ch);
            }
        }
    catch (NxsX_UnexpectedEOF & x)
        {
        x.msg << " (end-of-file inside {} braced statement)";
        eofAllowed = formerEOFAllowed;
        throw x;
        }
    eofAllowed = formerEOFAllowed ;
    }

void NxsToken::GetDoubleQuotedToken()
    {
    bool formerEOFAllowed = eofAllowed;
    eofAllowed = false;
    try
        {
        for(;;)
            {
            char ch = GetNextChar();
            if (ch == '\"')
                break;
            else
                AppendToToken(ch);
            }
        }
    catch (NxsX_UnexpectedEOF & x)
        {
        x.msg << " (end-of-file inside \" quoted statement)";
        eofAllowed = formerEOFAllowed;
        throw x;
        }
    eofAllowed = formerEOFAllowed ;
    }

void NxsToken::GetQuoted()
    {
    bool formerEOFAllowed = eofAllowed;
    eofAllowed = false;
    long fl = fileLine;
    long fc = fileColumn;

    try
        {
        for(;;)
            {
            char ch = GetNextChar();
            if (ch == '\'')
                {
                ch = GetNextChar();
                if (ch == '\'')
                    AppendToToken(ch);
                else
                    {
                    saved = ch;
                    break;
                    }
                }
            else
                AppendToToken(ch);
            }
        }
    catch (NxsX_UnexpectedEOF & x)
        {
        x.msg << " (end-of-file inside \' quoted token that started on line " << fl<< ", column " <<fc << ')';
        eofAllowed = formerEOFAllowed;
        throw x;
        }
    eofAllowed = formerEOFAllowed ;
    }

void NxsToken::GetParentheticalToken()
    {
    // Set level to 1 initially.  Every ')' encountered reduces
    // level by one, so that we know we can stop when level becomes 0.
    //
    int level = 1;
    std::vector<NxsComment> prevEmbedded = embeddedComments;
    embeddedComments.clear();
    char ch;
    ch = GetNextChar();
    for(;;)
        {
        if (atEOF)
            break;

        if (ch == '\'')
            {
            AppendToToken('\'');
            GetQuoted();
            AppendToToken('\'');
            ch = saved;
            saved = '\0';
            if (atEOF)
                {
                if (ch == ')' && level == 1)
                    {
                    AppendToToken(')');
                    break;
                    }
                else
                    {
                    NxsX_UnexpectedEOF x(*this);
                    x.msg << "(end-of-file inside () statement)";
                    }
                }
            continue;
            }
        if (ch == '[')
            {
            GetComment();
            assert(embeddedComments.size() == 1);
            AppendToToken('[');
            const std::string & body =  embeddedComments[0].GetText();
            token.append(body.begin(), body.end());
            AppendToToken(']');
            embeddedComments.clear();

            }
        else
            {
            if (ch == ')')
                level--;
            else if (ch == '(')
                level++;

            AppendToToken(ch);
            }

        if (level == 0)
            break;
        ch = GetNextChar();
        }
    embeddedComments = prevEmbedded;
    }

bool NxsToken::Abbreviation(
  NxsString s)  /* the comparison string */
    {
    int k;
    int slen = (int)s.size();
    int tlen = (int)token.size();
    char tokenChar, otherChar;

    // The variable mlen refers to the "mandatory" portion
    // that is the upper-case portion of s
    //
    int mlen;
    for (mlen = 0; mlen < slen; mlen++)
        {
        if (!isupper(s[mlen]))
            break;
        }

    // User must have typed at least mlen characters in
    // for there to even be a chance at a match
    //
    if (tlen < mlen)
        return false;

    // If user typed in more characters than are contained in s,
    // then there must be a mismatch
    //
    if (tlen > slen)
        return false;

    // Check the mandatory portion for mismatches
    //
    for (k = 0; k < mlen; k++)
        {
        tokenChar = (char)toupper( token[k]);
        otherChar = s[k];
        if (tokenChar != otherChar)
            return false;
        }

    // Check the auxiliary portion for mismatches (if necessary)
    //
    for (k = mlen; k < tlen; k++)
        {
        tokenChar = (char)toupper( token[k]);
        otherChar = (char)toupper( s[k]);
        if (tokenChar != otherChar)
            return false;
        }

    return true;
    }

bool NxsToken::Begins(
  NxsString s,          /* the comparison string */
  bool respect_case)    /* determines whether comparison is case sensitive */
    {
    unsigned k;
    char tokenChar, otherChar;

    unsigned slen = (unsigned)s.size();
    if (slen > token.size())
        return false;

    for (k = 0; k < slen; k++)
        {
        if (respect_case)
            {
            tokenChar = token[k];
            otherChar = s[k];
            }
        else
            {
            tokenChar = (char)toupper( token[k]);
            otherChar = (char)toupper( s[k]);
            }

        if (tokenChar != otherChar)
            return false;
        }

    return true;
    }

void NxsToken::GetNextToken()
    {
    ResetToken();

    char ch = ' ';
    if (saved == '\0' || IsWhitespace(saved))
        {
        // Skip leading whitespace

        while( IsWhitespace(ch) && !atEOF)
            ch = GetNextChar();
        saved = ch;
        }

    for(;;)
        {
        // Break now if singleCharacterToken mode on and token length > 0.
        //
        if (labileFlags & singleCharacterToken && !token.empty())
            break;

        // Get next character either from saved or from input stream.
        //
        if (saved != '\0')
            {
            ch = saved;
            saved = '\0';
            }
        else
            ch = GetNextChar();

        // Break now if we've hit EOF.
        //
        if (atEOF)
            break;
        if (strchr("\n\r \t", ch) != NULL)
            {
            if (ch == '\n' && labileFlags & newlineIsToken)
                {
                if (token.empty())
                    {
                    atEOL = 1;
                    AppendToToken(ch);
                    }
                else
                    {
                    // Newline came after token, save newline until next time when it will be
                    // reported as a separate token.
                    //
                    atEOL = 0;
                    saved = ch;
                    }
                break;
                }
            else
                {
                // Break only if we've begun adding to token (remember, if we hit a comment before a token,
                // there might be further white space between the comment and the next token).
                //
                if (!token.empty())
                    break;
                }
            }
        else if (ch == '_')
            {
            // If underscores are discovered in unquoted tokens, they should be
            // automatically converted to spaces.
            //
            if (!(labileFlags & preserveUnderscores))
                ch = ' ';
            AppendToToken(ch);
            }

        else if (ch == '[')
            {
            // Get rest of comment and deal with it, but notice that we only break if the comment ends a token,
            // not if it starts one (comment counts as whitespace). In the case of command comments
            // (if saveCommandComment) GetComment will add to the token NxsString, causing us to break because
            // token.size() will be greater than 0.
            comment.clear();
            if (GetComment())
                break;
            }
        else if (IsPunctuation(ch))
            {
            if (ch == '(' && (labileFlags & parentheticalToken))
                {
                AppendToToken(ch);
                GetParentheticalToken();
                }
            else if (ch == '{' && (labileFlags & curlyBracketedToken))
                {
                AppendToToken(ch);
                GetCurlyBracketedToken();
                }
            else if (ch == '\"' && (labileFlags & doubleQuotedToken))
                GetDoubleQuotedToken();
            else if (ch == '\'' && token.empty())
                GetQuoted();
            else
                {
                //save if we have started a token, consider the punctuation to
                // be the full token.
                if (token.size() > 0)
                    saved = ch;
                else
                    AppendToToken(ch);
                }
            break;
            }
        else
            AppendToToken(ch);
        }

    labileFlags = 0;
    }

void NxsToken::StripWhitespace()
    {
    NxsString s;
    for (unsigned j = 0; j < token.size(); j++)
        {
        if (IsWhitespace( token[j]))
            continue;
        s += token[j];
        }
    token = s;
    }

void NxsToken::ToUpper()
    {
    for (unsigned i = 0; i < token.size(); i++)
        token[i] = (char)toupper(token[i]);
    }

---