You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
938 lines
25 KiB
938 lines
25 KiB
ANTLR_BEGIN_NAMESPACE()
|
|
|
|
template<class ImplTraits>
|
|
TokenSource<ImplTraits>::TokenSource()
|
|
:m_eofToken( ImplTraits::CommonTokenType::TOKEN_EOF),
|
|
m_skipToken( ImplTraits::CommonTokenType::TOKEN_INVALID)
|
|
{
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE typename TokenSource<ImplTraits>::CommonTokenType& TokenSource<ImplTraits>::get_eofToken()
|
|
{
|
|
return m_eofToken;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE const typename TokenSource<ImplTraits>::TokenType& TokenSource<ImplTraits>::get_eofToken() const
|
|
{
|
|
return m_eofToken;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE typename TokenSource<ImplTraits>::CommonTokenType& TokenSource<ImplTraits>::get_skipToken()
|
|
{
|
|
return m_skipToken;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE typename TokenSource<ImplTraits>::StringType& TokenSource<ImplTraits>::get_fileName()
|
|
{
|
|
return m_fileName;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE void TokenSource<ImplTraits>::set_fileName( const StringType& fileName )
|
|
{
|
|
m_fileName = fileName;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenSource<ImplTraits>::LexerType* TokenSource<ImplTraits>::get_super()
|
|
{
|
|
return static_cast<LexerType*>(this);
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenSource<ImplTraits>::TokenType* TokenSource<ImplTraits>::nextTokenStr()
|
|
{
|
|
typedef typename LexerType::RecognizerSharedStateType RecognizerSharedStateType;
|
|
typedef typename LexerType::InputStreamType InputStreamType;
|
|
typedef typename LexerType::IntStreamType IntStreamType;
|
|
LexerType* lexer;
|
|
RecognizerSharedStateType* state;
|
|
InputStreamType* input;
|
|
IntStreamType* istream;
|
|
|
|
lexer = this->get_super();
|
|
state = lexer->get_rec()->get_state();
|
|
input = lexer->get_input();
|
|
istream = input->get_istream();
|
|
|
|
/// Loop until we get a non skipped token or EOF
|
|
///
|
|
for (;;)
|
|
{
|
|
// Get rid of any previous token (token factory takes care of
|
|
// any de-allocation when this token is finally used up.
|
|
//
|
|
state->set_token_present(false);
|
|
state->set_error(false); // Start out without an exception
|
|
state->set_failed(false);
|
|
|
|
// Now call the matching rules and see if we can generate a new token
|
|
//
|
|
for (;;)
|
|
{
|
|
// Record the start of the token in our input stream.
|
|
//
|
|
state->set_channel( TOKEN_DEFAULT_CHANNEL );
|
|
state->set_tokenStartCharIndex( (ANTLR_MARKER)input->get_nextChar() );
|
|
state->set_tokenStartCharPositionInLine( input->get_charPositionInLine() );
|
|
state->set_tokenStartLine( input->get_line() );
|
|
state->set_text("");
|
|
|
|
if (istream->_LA(1) == ANTLR_CHARSTREAM_EOF)
|
|
{
|
|
// Reached the end of the current stream, nothing more to do if this is
|
|
// the last in the stack.
|
|
//
|
|
TokenType& teof = m_eofToken;
|
|
|
|
teof.set_startIndex(lexer->getCharIndex());
|
|
teof.set_stopIndex(lexer->getCharIndex());
|
|
teof.set_line(lexer->getLine());
|
|
return &teof;
|
|
}
|
|
|
|
state->set_token_present( false );
|
|
state->set_error(false); // Start out without an exception
|
|
state->set_failed(false);
|
|
|
|
// Call the generated lexer, see if it can get a new token together.
|
|
//
|
|
lexer->mTokens();
|
|
|
|
if (state->get_error() == true)
|
|
{
|
|
// Recognition exception, report it and try to recover.
|
|
//
|
|
state->set_failed(true);
|
|
lexer->get_rec()->reportError();
|
|
lexer->recover();
|
|
}
|
|
else
|
|
{
|
|
if ( !state->get_token_present() )
|
|
{
|
|
// Emit the real token, which adds it in to the token stream basically
|
|
//
|
|
lexer->emit();
|
|
}
|
|
else if ( *(state->get_token()) == m_skipToken )
|
|
{
|
|
// A real token could have been generated, but "Computer say's naaaaah" and it
|
|
// it is just something we need to skip altogether.
|
|
//
|
|
continue;
|
|
}
|
|
|
|
// Good token, not skipped, not EOF token
|
|
//
|
|
return state->get_token();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenSource<ImplTraits>::TokenType* TokenSource<ImplTraits>::nextToken()
|
|
{
|
|
return this->nextToken( BoolForwarder<LexerType::IsFiltered>() );
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenSource<ImplTraits>::CommonTokenType* TokenSource<ImplTraits>::nextToken( BoolForwarder<true> /*isFiltered*/ )
|
|
{
|
|
LexerType* lexer;
|
|
typename LexerType::RecognizerSharedStateType* state;
|
|
|
|
lexer = this->get_super();
|
|
state = lexer->get_lexstate();
|
|
|
|
/* Get rid of any previous token (token factory takes care of
|
|
* any deallocation when this token is finally used up.
|
|
*/
|
|
state->set_token_present( false );
|
|
state->set_error( false ); /* Start out without an exception */
|
|
state->set_failed(false);
|
|
|
|
/* Record the start of the token in our input stream.
|
|
*/
|
|
state->set_tokenStartCharIndex( lexer->index() );
|
|
state->set_tokenStartCharPositionInLine( lexer->getCharPositionInLine() );
|
|
state->set_tokenStartLine( lexer->getLine() );
|
|
state->set_text("");
|
|
|
|
/* Now call the matching rules and see if we can generate a new token
|
|
*/
|
|
for (;;)
|
|
{
|
|
if (lexer->LA(1) == ANTLR_CHARSTREAM_EOF)
|
|
{
|
|
/* Reached the end of the stream, nothing more to do.
|
|
*/
|
|
CommonTokenType& teof = m_eofToken;
|
|
|
|
teof.set_startIndex(lexer->getCharIndex());
|
|
teof.set_stopIndex(lexer->getCharIndex());
|
|
teof.set_line(lexer->getLine());
|
|
return &teof;
|
|
}
|
|
|
|
state->set_token_present(false);
|
|
state->set_error(false); /* Start out without an exception */
|
|
|
|
{
|
|
ANTLR_MARKER m;
|
|
|
|
m = lexer->get_istream()->mark();
|
|
state->set_backtracking(1); /* No exceptions */
|
|
state->set_failed(false);
|
|
|
|
/* Call the generated lexer, see if it can get a new token together.
|
|
*/
|
|
lexer->mTokens();
|
|
state->set_backtracking(0);
|
|
|
|
/* mTokens backtracks with synpred at BACKTRACKING==2
|
|
and we set the synpredgate to allow actions at level 1. */
|
|
|
|
if(state->get_failed())
|
|
{
|
|
lexer->rewind(m);
|
|
lexer->consume(); //<! advance one char and try again !>
|
|
}
|
|
else
|
|
{
|
|
lexer->emit(); /* Assemble the token and emit it to the stream */
|
|
TokenType* tok = state->get_token();
|
|
return tok;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenSource<ImplTraits>::CommonTokenType* TokenSource<ImplTraits>::nextToken( BoolForwarder<false> /*isFiltered*/ )
|
|
{
|
|
// Find the next token in the current stream
|
|
//
|
|
CommonTokenType* tok = this->nextTokenStr();
|
|
|
|
// If we got to the EOF token then switch to the previous
|
|
// input stream if there were any and just return the
|
|
// EOF if there are none. We must check the next token
|
|
// in any outstanding input stream we pop into the active
|
|
// role to see if it was sitting at EOF after PUSHing the
|
|
// stream we just consumed, otherwise we will return EOF
|
|
// on the reinstalled input stream, when in actual fact
|
|
// there might be more input streams to POP before the
|
|
// real EOF of the whole logical inptu stream. Hence we
|
|
// use a while loop here until we find somethign in the stream
|
|
// that isn't EOF or we reach the actual end of the last input
|
|
// stream on the stack.
|
|
//
|
|
while(tok->get_type() == CommonTokenType::TOKEN_EOF)
|
|
{
|
|
typename ImplTraits::LexerType* lexer;
|
|
lexer = static_cast<typename ImplTraits::LexerType*>( this->get_super() );
|
|
|
|
if ( lexer->get_rec()->get_state()->get_streams().size() > 0)
|
|
{
|
|
// We have another input stream in the stack so we
|
|
// need to revert to it, then resume the loop to check
|
|
// it wasn't sitting at EOF itself.
|
|
//
|
|
lexer->popCharStream();
|
|
tok = this->nextTokenStr();
|
|
}
|
|
else
|
|
{
|
|
// There were no more streams on the input stack
|
|
// so this EOF is the 'real' logical EOF for
|
|
// the input stream. So we just exit the loop and
|
|
// return the EOF we have found.
|
|
//
|
|
break;
|
|
}
|
|
|
|
}
|
|
|
|
// return whatever token we have, which may be EOF
|
|
//
|
|
return tok;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
TokenStream<ImplTraits>::TokenStream()
|
|
{
|
|
m_tokenSource = NULL;
|
|
m_debugger = NULL;
|
|
m_initialStreamState = false;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenStream<ImplTraits>::IntStreamType* TokenStream<ImplTraits>::get_istream()
|
|
{
|
|
return this;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
TokenStream<ImplTraits>::TokenStream(TokenSourceType* source, DebugEventListenerType* debugger)
|
|
{
|
|
m_initialStreamState = false;
|
|
m_tokenSource = source;
|
|
m_debugger = debugger;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
CommonTokenStream<ImplTraits>::CommonTokenStream(ANTLR_UINT32 , TokenSourceType* source,
|
|
DebugEventListenerType* debugger)
|
|
: CommonTokenStream<ImplTraits>::BaseType( source, debugger )
|
|
{
|
|
m_p = -1;
|
|
m_channel = TOKEN_DEFAULT_CHANNEL;
|
|
m_discardOffChannel = false;
|
|
m_nissued = 0;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename CommonTokenStream<ImplTraits>::TokensType& CommonTokenStream<ImplTraits>::get_tokens()
|
|
{
|
|
return m_tokens;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
const typename CommonTokenStream<ImplTraits>::TokensType& CommonTokenStream<ImplTraits>::get_tokens() const
|
|
{
|
|
return m_tokens;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename CommonTokenStream<ImplTraits>::DiscardSetType& CommonTokenStream<ImplTraits>::get_discardSet()
|
|
{
|
|
return m_discardSet;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
const typename CommonTokenStream<ImplTraits>::DiscardSetType& CommonTokenStream<ImplTraits>::get_discardSet() const
|
|
{
|
|
return m_discardSet;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE ANTLR_INT32 CommonTokenStream<ImplTraits>::get_p() const
|
|
{
|
|
return m_p;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE void CommonTokenStream<ImplTraits>::set_p( ANTLR_INT32 p )
|
|
{
|
|
m_p = p;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE void CommonTokenStream<ImplTraits>::inc_p()
|
|
{
|
|
++m_p;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE void CommonTokenStream<ImplTraits>::dec_p()
|
|
{
|
|
--m_p;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_INLINE ANTLR_MARKER CommonTokenStream<ImplTraits>::index_impl()
|
|
{
|
|
return m_p;
|
|
}
|
|
|
|
// Reset a token stream so it can be used again and can reuse it's
|
|
// resources.
|
|
//
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::reset()
|
|
{
|
|
// Free any resources that ar most like specifc to the
|
|
// run we just did.
|
|
//
|
|
m_discardSet.clear();
|
|
m_channelOverrides.clear();
|
|
|
|
// Now, if there were any existing tokens in the stream,
|
|
// then we just reset the vector count so that it starts
|
|
// again. We must traverse the entries unfortunately as
|
|
// there may be free pointers for custom token types and
|
|
// so on. However that is just a quick NULL check on the
|
|
// vector entries.
|
|
//
|
|
m_tokens.clear();
|
|
|
|
// Reset to defaults
|
|
//
|
|
m_discardOffChannel = false;
|
|
m_channel = ImplTraits::CommonTokenType::TOKEN_DEFAULT_CHANNEL;
|
|
m_p = -1;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void TokenStream<ImplTraits>::setDebugListener(DebugEventListenerType* debugger)
|
|
{
|
|
m_debugger = debugger;
|
|
m_initialStreamState = false;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
const typename TokenStream<ImplTraits>::TokenType* TokenStream<ImplTraits>::_LT(ANTLR_INT32 k)
|
|
{
|
|
ANTLR_INT32 i;
|
|
ANTLR_INT32 n;
|
|
TokenStreamType* cts;
|
|
|
|
cts = this->get_super();
|
|
|
|
if(k < 0)
|
|
{
|
|
return cts->LB(-k);
|
|
}
|
|
|
|
ANTLR_INT32 req_idx = cts->get_p() + k - 1;
|
|
ANTLR_INT32 cached_size = static_cast<ANTLR_INT32>(this->get_istream()->get_cachedSize());
|
|
|
|
if( (cts->get_p() == -1) ||
|
|
( ( req_idx >= cached_size ) && ( (cached_size % ImplTraits::TOKEN_FILL_BUFFER_INCREMENT) == 0 ) )
|
|
)
|
|
{
|
|
cts->fillBuffer();
|
|
}
|
|
|
|
// Here we used to check for k == 0 and return 0, but this seems
|
|
// a superfluous check to me. LT(k=0) is therefore just undefined
|
|
// and we won't waste the clock cycles on the check
|
|
//
|
|
cached_size = static_cast<ANTLR_INT32>(this->get_istream()->get_cachedSize());
|
|
if ( req_idx >= cached_size )
|
|
{
|
|
TokenType& teof = cts->get_tokenSource()->get_eofToken();
|
|
|
|
teof.set_startIndex( this->get_istream()->index());
|
|
teof.set_stopIndex( this->get_istream()->index());
|
|
return &teof;
|
|
}
|
|
|
|
i = cts->get_p();
|
|
n = 1;
|
|
|
|
/* Need to find k good tokens, skipping ones that are off channel
|
|
*/
|
|
while( n < k)
|
|
{
|
|
/* Skip off-channel tokens */
|
|
i = cts->skipOffTokenChannels(i+1); /* leave p on valid token */
|
|
n++;
|
|
}
|
|
|
|
if( ( i >= cached_size ) && ( (cached_size % ImplTraits::TOKEN_FILL_BUFFER_INCREMENT) == 0 ) )
|
|
{
|
|
cts->fillBuffer();
|
|
}
|
|
if ( (ANTLR_UINT32) i >= this->get_istream()->get_cachedSize() )
|
|
{
|
|
TokenType& teof = cts->get_tokenSource()->get_eofToken();
|
|
|
|
teof.set_startIndex(this->get_istream()->index());
|
|
teof.set_stopIndex(this->get_istream()->index());
|
|
return &teof;
|
|
}
|
|
|
|
// Here the token must be in the input vector. Rather then incur
|
|
// function call penalty, we just return the pointer directly
|
|
// from the vector
|
|
//
|
|
return cts->getToken(i);
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::LB(ANTLR_INT32 k)
|
|
{
|
|
ANTLR_INT32 i;
|
|
ANTLR_INT32 n;
|
|
|
|
if (m_p == -1)
|
|
{
|
|
this->fillBuffer();
|
|
}
|
|
if (k == 0)
|
|
{
|
|
return NULL;
|
|
}
|
|
if ((m_p - k) < 0)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
i = m_p;
|
|
n = 1;
|
|
|
|
/* Need to find k good tokens, going backwards, skipping ones that are off channel
|
|
*/
|
|
while (n <= k)
|
|
{
|
|
/* Skip off-channel tokens
|
|
*/
|
|
|
|
i = this->skipOffTokenChannelsReverse(i - 1); /* leave p on valid token */
|
|
n++;
|
|
}
|
|
if (i < 0)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
// Here the token must be in the input vector. Rather then incut
|
|
// function call penalty, we jsut return the pointer directly
|
|
// from the vector
|
|
//
|
|
return this->getToken(i);
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::getToken(ANTLR_MARKER i)
|
|
{
|
|
return this->get(i);
|
|
}
|
|
|
|
|
|
template<class ImplTraits>
|
|
const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::get(ANTLR_MARKER i)
|
|
{
|
|
return this->getToken( static_cast<ANTLR_MARKER>(i),
|
|
BoolForwarder<ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE>() );
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::getToken( ANTLR_MARKER tok_idx,
|
|
BoolForwarder<true> /*tokens_accessed_from_owning_rule*/ )
|
|
{
|
|
typename TokensType::iterator iter = m_tokens.find(tok_idx);
|
|
if( iter == m_tokens.end() )
|
|
{
|
|
TokenAccessException ex;
|
|
throw ex;
|
|
}
|
|
const TokenType& tok = iter->second;
|
|
return &tok;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::getToken( ANTLR_MARKER tok_idx, BoolForwarder<false> /*tokens_accessed_from_owning_rule*/ )
|
|
{
|
|
TokenType& tok = m_tokens.at( static_cast<ANTLR_UINT32>(tok_idx) );
|
|
return &tok;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenStream<ImplTraits>::TokenSourceType* TokenStream<ImplTraits>::get_tokenSource() const
|
|
{
|
|
return m_tokenSource;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void TokenStream<ImplTraits>::set_tokenSource( TokenSourceType* tokenSource )
|
|
{
|
|
m_tokenSource = tokenSource;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenStream<ImplTraits>::StringType TokenStream<ImplTraits>::toString()
|
|
{
|
|
TokenStreamType* cts = static_cast<TokenStreamType>(this);
|
|
|
|
if (cts->get_p() == -1)
|
|
{
|
|
cts->fillBuffer();
|
|
}
|
|
|
|
return this->toStringSS(0, this->get_istream()->size());
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenStream<ImplTraits>::StringType
|
|
TokenStream<ImplTraits>::toStringSS(ANTLR_MARKER start, ANTLR_MARKER stop)
|
|
{
|
|
StringType string;
|
|
TokenSourceType* tsource;
|
|
const TokenType* tok;
|
|
TokenStreamType* cts;
|
|
|
|
cts = this->get_super();
|
|
|
|
if (cts->get_p() == -1)
|
|
{
|
|
cts->fillBuffer();
|
|
}
|
|
if (stop >= this->get_istream()->size())
|
|
{
|
|
stop = this->get_istream()->size() - 1;
|
|
}
|
|
|
|
/* Who is giving us these tokens?
|
|
*/
|
|
tsource = cts->get_tokenSource();
|
|
|
|
if (tsource != NULL && !cts->get_tokens().empty() )
|
|
{
|
|
/* Finally, let's get a string
|
|
*/
|
|
for (ANTLR_MARKER i = start; i <= stop; i++)
|
|
{
|
|
tok = cts->get(i);
|
|
if (tok != NULL)
|
|
{
|
|
string.append( tok->getText() );
|
|
}
|
|
}
|
|
|
|
return string;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename TokenStream<ImplTraits>::StringType
|
|
TokenStream<ImplTraits>::toStringTT(const TokenType* start, const TokenType* stop)
|
|
{
|
|
if (start != NULL && stop != NULL)
|
|
{
|
|
return this->toStringSS( start->get_tokenIndex(),
|
|
stop->get_tokenIndex());
|
|
}
|
|
else
|
|
{
|
|
return "";
|
|
}
|
|
}
|
|
|
|
/** A simple filter mechanism whereby you can tell this token stream
|
|
* to force all tokens of type ttype to be on channel. For example,
|
|
* when interpreting, we cannot execute actions so we need to tell
|
|
* the stream to force all WS and NEWLINE to be a different, ignored,
|
|
* channel.
|
|
*/
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::setTokenTypeChannel ( ANTLR_UINT32 ttype, ANTLR_UINT32 channel)
|
|
{
|
|
/* We add one to the channel so we can distinguish NULL as being no entry in the
|
|
* table for a particular token type.
|
|
*/
|
|
m_channelOverrides[ttype] = (ANTLR_UINT32)channel + 1;
|
|
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::discardTokenType(ANTLR_INT32 ttype)
|
|
{
|
|
/* We add one to the channel so we can distinguish NULL as being no entry in the
|
|
* table for a particular token type. We could use bitsets for this I suppose too.
|
|
*/
|
|
m_discardSet.insert(ttype);
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::discardOffChannelToks(bool discard)
|
|
{
|
|
m_discardOffChannel = discard;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
typename CommonTokenStream<ImplTraits>::TokensType* CommonTokenStream<ImplTraits>::getTokens()
|
|
{
|
|
if (m_p == -1)
|
|
{
|
|
this->fillBuffer();
|
|
}
|
|
|
|
return &m_tokens;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::getTokenRange(ANTLR_UINT32 start, ANTLR_UINT32 stop,
|
|
TokensListType& tokenRange)
|
|
{
|
|
return this->getTokensSet(start, stop, NULL, tokenRange);
|
|
}
|
|
|
|
/** Given a start and stop index, return a List of all tokens in
|
|
* the token type BitSet. Return null if no tokens were found. This
|
|
* method looks at both on and off channel tokens.
|
|
*/
|
|
template<class ImplTraits>
|
|
void
|
|
CommonTokenStream<ImplTraits>::getTokensSet(ANTLR_UINT32 start, ANTLR_UINT32 stop, BitsetType* types,
|
|
TokensListType& filteredList )
|
|
{
|
|
ANTLR_UINT32 i;
|
|
ANTLR_UINT32 n;
|
|
TokenType* tok;
|
|
|
|
if ( m_p == -1)
|
|
{
|
|
this->fillBuffer();
|
|
}
|
|
if (stop > this->get_istream()->size())
|
|
{
|
|
stop = this->get_istream()->size();
|
|
}
|
|
if (start > stop)
|
|
{
|
|
return;
|
|
}
|
|
|
|
/* We have the range set, now we need to iterate through the
|
|
* installed tokens and create a new list with just the ones we want
|
|
* in it. We are just moving pointers about really.
|
|
*/
|
|
for(i = start, n = 0; i<= stop; i++)
|
|
{
|
|
tok = this->get(i);
|
|
|
|
if ( types == NULL
|
|
|| (types->isMember( tok->get_type() ) == true )
|
|
)
|
|
{
|
|
filteredList.push_back(tok);
|
|
}
|
|
}
|
|
|
|
return ;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void
|
|
CommonTokenStream<ImplTraits>::getTokensList(ANTLR_UINT32 start, ANTLR_UINT32 stop,
|
|
const IntListType& list, TokensListType& newlist)
|
|
{
|
|
BitsetType* bitSet;
|
|
|
|
bitSet = Bitset<ImplTraits>::BitsetFromList(list);
|
|
this->getTokensSet(start, stop, bitSet, newlist);
|
|
delete bitSet;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void
|
|
CommonTokenStream<ImplTraits>::getTokensType(ANTLR_UINT32 start, ANTLR_UINT32 stop, ANTLR_UINT32 type,
|
|
TokensListType& newlist )
|
|
{
|
|
BitsetType* bitSet;
|
|
|
|
bitSet = BitsetType::BitsetOf(type, -1);
|
|
this->getTokensSet(start, stop, bitSet, newlist);
|
|
|
|
delete bitSet;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::fillBufferExt()
|
|
{
|
|
this->fillBuffer();
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
bool CommonTokenStream<ImplTraits>::hasReachedFillbufferTarget( ANTLR_UINT32 cnt,
|
|
BoolForwarder<true> )
|
|
{
|
|
return ( cnt >= ImplTraits::TOKEN_FILL_BUFFER_INCREMENT );
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
bool CommonTokenStream<ImplTraits>::hasReachedFillbufferTarget( ANTLR_UINT32,
|
|
BoolForwarder<false> )
|
|
{
|
|
return false;
|
|
}
|
|
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::fillBuffer()
|
|
{
|
|
ANTLR_UINT32 index;
|
|
TokenType* tok;
|
|
bool discard;
|
|
|
|
/* Start at index 0 of course
|
|
*/
|
|
ANTLR_UINT32 cached_p = (m_p < 0) ? 0 : m_p;
|
|
index = m_nissued;
|
|
ANTLR_UINT32 cnt = 0;
|
|
|
|
/* Pick out the next token from the token source
|
|
* Remember we just get a pointer (reference if you like) here
|
|
* and so if we store it anywhere, we don't set any pointers to auto free it.
|
|
*/
|
|
tok = this->get_tokenSource()->nextToken();
|
|
|
|
while ( tok->get_type() != TokenType::TOKEN_EOF )
|
|
{
|
|
discard = false; /* Assume we are not discarding */
|
|
|
|
/* I employ a bit of a trick, or perhaps hack here. Rather than
|
|
* store a pointer to a structure in the override map and discard set
|
|
* we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
|
|
* we can distinguish "not being there" from "being channel or type 0"
|
|
*/
|
|
|
|
if ( m_discardSet.find(tok->get_type()) != m_discardSet.end() )
|
|
{
|
|
discard = true;
|
|
}
|
|
else if ( m_discardOffChannel == true
|
|
&& tok->get_channel() != m_channel
|
|
)
|
|
{
|
|
discard = true;
|
|
}
|
|
else if (!m_channelOverrides.empty())
|
|
{
|
|
/* See if this type is in the override map
|
|
*/
|
|
typename ChannelOverridesType::iterator iter = m_channelOverrides.find( tok->get_type() + 1 );
|
|
|
|
if (iter != m_channelOverrides.end())
|
|
{
|
|
/* Override found
|
|
*/
|
|
tok->set_channel( ANTLR_UINT32_CAST(iter->second) - 1);
|
|
}
|
|
}
|
|
|
|
/* If not discarding it, add it to the list at the current index
|
|
*/
|
|
if (discard == false)
|
|
{
|
|
/* Add it, indicating that we will delete it and the table should not
|
|
*/
|
|
tok->set_tokenIndex(index);
|
|
++m_p;
|
|
this->insertToken(*tok);
|
|
index++;
|
|
m_nissued++;
|
|
cnt++;
|
|
}
|
|
|
|
if( !this->hasReachedFillbufferTarget( cnt,
|
|
BoolForwarder<ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE>() ) )
|
|
tok = this->get_tokenSource()->nextToken();
|
|
else
|
|
break;
|
|
}
|
|
|
|
/* Cache the size so we don't keep doing indirect method calls. We do this as
|
|
* early as possible so that anything after this may utilize the cached value.
|
|
*/
|
|
this->get_istream()->set_cachedSize( m_nissued );
|
|
|
|
/* Set the consume pointer to the first token that is on our channel, we just read
|
|
*/
|
|
m_p = cached_p;
|
|
m_p = this->skipOffTokenChannels( m_p );
|
|
|
|
}
|
|
/// Given a starting index, return the index of the first on-channel
|
|
/// token.
|
|
///
|
|
template<class ImplTraits>
|
|
ANTLR_UINT32 CommonTokenStream<ImplTraits>::skipOffTokenChannels(ANTLR_INT32 i)
|
|
{
|
|
ANTLR_INT32 n;
|
|
n = this->get_istream()->get_cachedSize();
|
|
|
|
while (i < n)
|
|
{
|
|
const TokenType* tok = this->getToken(i);
|
|
|
|
if (tok->get_channel() != m_channel )
|
|
{
|
|
i++;
|
|
}
|
|
else
|
|
{
|
|
return i;
|
|
}
|
|
}
|
|
return i;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
ANTLR_UINT32 CommonTokenStream<ImplTraits>::skipOffTokenChannelsReverse(ANTLR_INT32 x)
|
|
{
|
|
while (x >= 0)
|
|
{
|
|
const TokenType* tok = this->getToken(x);
|
|
|
|
if( tok->get_channel() != m_channel )
|
|
{
|
|
x--;
|
|
}
|
|
else
|
|
{
|
|
return x;
|
|
}
|
|
}
|
|
return x;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::discardTokens( ANTLR_MARKER start, ANTLR_MARKER stop )
|
|
{
|
|
this->discardTokens( start, stop, BoolForwarder< ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE >() );
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::discardTokens( ANTLR_MARKER start, ANTLR_MARKER stop,
|
|
BoolForwarder<true> /*tokens_accessed_from_owning_rule */ )
|
|
{
|
|
typename TokensType::iterator iter1 = m_tokens.lower_bound(start);
|
|
typename TokensType::iterator iter2 = m_tokens.upper_bound(stop);
|
|
m_tokens.erase( iter1, iter2 );
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::discardTokens( ANTLR_MARKER start, ANTLR_MARKER stop,
|
|
BoolForwarder<false> /*tokens_accessed_from_owning_rule*/ )
|
|
{
|
|
m_tokens.erase( m_tokens.begin() + start, m_tokens.begin() + stop );
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::insertToken( const TokenType& tok )
|
|
{
|
|
this->insertToken( tok, BoolForwarder< ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE >() );
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::insertToken( const TokenType& tok, BoolForwarder<true> /*tokens_accessed_from_owning_rule*/ )
|
|
{
|
|
assert( m_tokens.find( tok.get_index() ) == m_tokens.end() );
|
|
assert( tok.get_index() == m_nissued );
|
|
m_tokens[ tok.get_index() ] = tok;
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
void CommonTokenStream<ImplTraits>::insertToken( const TokenType& tok, BoolForwarder<false> /*tokens_accessed_from_owning_rule*/ )
|
|
{
|
|
m_tokens.push_back( tok );
|
|
}
|
|
|
|
template<class ImplTraits>
|
|
CommonTokenStream<ImplTraits>::~CommonTokenStream()
|
|
{
|
|
m_tokens.clear();
|
|
}
|
|
|
|
ANTLR_END_NAMESPACE()
|