You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
527 lines
16 KiB
527 lines
16 KiB
/*
|
|
* [The "BSD licence"]
|
|
* Copyright (c) 2005-2008 Terence Parr
|
|
* All rights reserved.
|
|
*
|
|
* Conversion to C#:
|
|
* Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* 3. The name of the author may not be used to endorse or promote products
|
|
* derived from this software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
namespace Antlr.Runtime
|
|
{
|
|
using System.Collections.Generic;
|
|
|
|
using InvalidOperationException = System.InvalidOperationException;
|
|
using StringBuilder = System.Text.StringBuilder;
|
|
|
|
/** <summary>
|
|
* The most common stream of tokens is one where every token is buffered up
|
|
* and tokens are prefiltered for a certain channel (the parser will only
|
|
* see these tokens and cannot change the filter channel number during the
|
|
* parse).
|
|
* </summary>
|
|
*
|
|
* <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks>
|
|
*/
|
|
[System.Serializable]
|
|
public class LegacyCommonTokenStream : ITokenStream
|
|
{
|
|
[System.NonSerialized]
|
|
ITokenSource _tokenSource;
|
|
|
|
/** <summary>
|
|
* Record every single token pulled from the source so we can reproduce
|
|
* chunks of it later.
|
|
* </summary>
|
|
*/
|
|
protected List<IToken> tokens;
|
|
|
|
/** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */
|
|
protected IDictionary<int, int> channelOverrideMap;
|
|
|
|
/** <summary>Set<tokentype>; discard any tokens with this type</summary> */
|
|
protected List<int> discardSet;
|
|
|
|
/** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */
|
|
protected int channel = TokenChannels.Default;
|
|
|
|
/** <summary>By default, track all incoming tokens</summary> */
|
|
protected bool discardOffChannelTokens = false;
|
|
|
|
/** <summary>Track the last mark() call result value for use in rewind().</summary> */
|
|
protected int lastMarker;
|
|
|
|
/** <summary>
|
|
* The index into the tokens list of the current token (next token
|
|
* to consume). p==-1 indicates that the tokens list is empty
|
|
* </summary>
|
|
*/
|
|
protected int p = -1;
|
|
|
|
public LegacyCommonTokenStream()
|
|
{
|
|
tokens = new List<IToken>( 500 );
|
|
}
|
|
|
|
public LegacyCommonTokenStream(ITokenSource tokenSource)
|
|
: this()
|
|
{
|
|
this._tokenSource = tokenSource;
|
|
}
|
|
|
|
public LegacyCommonTokenStream( ITokenSource tokenSource, int channel )
|
|
: this( tokenSource )
|
|
{
|
|
this.channel = channel;
|
|
}
|
|
|
|
public virtual int Index
|
|
{
|
|
get
|
|
{
|
|
return p;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// How deep have we gone?
|
|
/// </summary>
|
|
public virtual int Range
|
|
{
|
|
get;
|
|
protected set;
|
|
}
|
|
|
|
/** <summary>Reset this token stream by setting its token source.</summary> */
|
|
public virtual void SetTokenSource( ITokenSource tokenSource )
|
|
{
|
|
this._tokenSource = tokenSource;
|
|
tokens.Clear();
|
|
p = -1;
|
|
channel = TokenChannels.Default;
|
|
}
|
|
|
|
/** <summary>
|
|
* Load all tokens from the token source and put in tokens.
|
|
* This is done upon first LT request because you might want to
|
|
* set some token type / channel overrides before filling buffer.
|
|
* </summary>
|
|
*/
|
|
public virtual void FillBuffer()
|
|
{
|
|
// fast return if the buffer is already full
|
|
if ( p != -1 )
|
|
return;
|
|
|
|
int index = 0;
|
|
IToken t = _tokenSource.NextToken();
|
|
while ( t != null && t.Type != CharStreamConstants.EndOfFile )
|
|
{
|
|
bool discard = false;
|
|
// is there a channel override for token type?
|
|
int channelI;
|
|
if ( channelOverrideMap != null && channelOverrideMap.TryGetValue( t.Type, out channelI ) )
|
|
t.Channel = channelI;
|
|
|
|
//if ( channelOverrideMap != null && channelOverrideMap.ContainsKey( t.getType() ) )
|
|
//{
|
|
// object channelI = channelOverrideMap.get( t.getType() );
|
|
// if ( channelI != null )
|
|
// {
|
|
// t.setChannel( (int)channelI );
|
|
// }
|
|
//}
|
|
if ( discardSet != null &&
|
|
discardSet.Contains( t.Type ) )
|
|
{
|
|
discard = true;
|
|
}
|
|
else if ( discardOffChannelTokens && t.Channel != this.channel )
|
|
{
|
|
discard = true;
|
|
}
|
|
if ( !discard )
|
|
{
|
|
t.TokenIndex = index;
|
|
tokens.Add( t );
|
|
index++;
|
|
}
|
|
t = _tokenSource.NextToken();
|
|
}
|
|
// leave p pointing at first token on channel
|
|
p = 0;
|
|
p = SkipOffTokenChannels( p );
|
|
}
|
|
|
|
/** <summary>
|
|
* Move the input pointer to the next incoming token. The stream
|
|
* must become active with LT(1) available. consume() simply
|
|
* moves the input pointer so that LT(1) points at the next
|
|
* input symbol. Consume at least one token.
|
|
* </summary>
|
|
*
|
|
* <remarks>
|
|
* Walk past any token not on the channel the parser is listening to.
|
|
* </remarks>
|
|
*/
|
|
public virtual void Consume()
|
|
{
|
|
if ( p < tokens.Count )
|
|
{
|
|
p++;
|
|
p = SkipOffTokenChannels( p ); // leave p on valid token
|
|
}
|
|
}
|
|
|
|
/** <summary>Given a starting index, return the index of the first on-channel token.</summary> */
|
|
protected virtual int SkipOffTokenChannels( int i )
|
|
{
|
|
int n = tokens.Count;
|
|
while ( i < n && ( (IToken)tokens[i] ).Channel != channel )
|
|
{
|
|
i++;
|
|
}
|
|
return i;
|
|
}
|
|
|
|
protected virtual int SkipOffTokenChannelsReverse( int i )
|
|
{
|
|
while ( i >= 0 && ( (IToken)tokens[i] ).Channel != channel )
|
|
{
|
|
i--;
|
|
}
|
|
return i;
|
|
}
|
|
|
|
/** <summary>
|
|
* A simple filter mechanism whereby you can tell this token stream
|
|
* to force all tokens of type ttype to be on channel. For example,
|
|
* when interpreting, we cannot exec actions so we need to tell
|
|
* the stream to force all WS and NEWLINE to be a different, ignored
|
|
* channel.
|
|
* </summary>
|
|
*/
|
|
public virtual void SetTokenTypeChannel( int ttype, int channel )
|
|
{
|
|
if ( channelOverrideMap == null )
|
|
{
|
|
channelOverrideMap = new Dictionary<int, int>();
|
|
}
|
|
channelOverrideMap[ttype] = channel;
|
|
}
|
|
|
|
public virtual void DiscardTokenType( int ttype )
|
|
{
|
|
if ( discardSet == null )
|
|
{
|
|
discardSet = new List<int>();
|
|
}
|
|
discardSet.Add( ttype );
|
|
}
|
|
|
|
public virtual void SetDiscardOffChannelTokens( bool discardOffChannelTokens )
|
|
{
|
|
this.discardOffChannelTokens = discardOffChannelTokens;
|
|
}
|
|
|
|
public virtual IList<IToken> GetTokens()
|
|
{
|
|
if ( p == -1 )
|
|
{
|
|
FillBuffer();
|
|
}
|
|
return tokens;
|
|
}
|
|
|
|
public virtual IList<IToken> GetTokens( int start, int stop )
|
|
{
|
|
return GetTokens( start, stop, (BitSet)null );
|
|
}
|
|
|
|
/** <summary>
|
|
* Given a start and stop index, return a List of all tokens in
|
|
* the token type BitSet. Return null if no tokens were found. This
|
|
* method looks at both on and off channel tokens.
|
|
* </summary>
|
|
*/
|
|
public virtual IList<IToken> GetTokens( int start, int stop, BitSet types )
|
|
{
|
|
if ( p == -1 )
|
|
{
|
|
FillBuffer();
|
|
}
|
|
if ( stop >= tokens.Count )
|
|
{
|
|
stop = tokens.Count - 1;
|
|
}
|
|
if ( start < 0 )
|
|
{
|
|
start = 0;
|
|
}
|
|
if ( start > stop )
|
|
{
|
|
return null;
|
|
}
|
|
|
|
// list = tokens[start:stop]:{Token t, t.getType() in types}
|
|
IList<IToken> filteredTokens = new List<IToken>();
|
|
for ( int i = start; i <= stop; i++ )
|
|
{
|
|
IToken t = tokens[i];
|
|
if ( types == null || types.Member( t.Type ) )
|
|
{
|
|
filteredTokens.Add( t );
|
|
}
|
|
}
|
|
if ( filteredTokens.Count == 0 )
|
|
{
|
|
filteredTokens = null;
|
|
}
|
|
return filteredTokens;
|
|
}
|
|
|
|
public virtual IList<IToken> GetTokens( int start, int stop, IList<int> types )
|
|
{
|
|
return GetTokens( start, stop, new BitSet( types ) );
|
|
}
|
|
|
|
public virtual IList<IToken> GetTokens( int start, int stop, int ttype )
|
|
{
|
|
return GetTokens( start, stop, BitSet.Of( ttype ) );
|
|
}
|
|
|
|
/** <summary>
|
|
* Get the ith token from the current position 1..n where k=1 is the
|
|
* first symbol of lookahead.
|
|
* </summary>
|
|
*/
|
|
public virtual IToken LT( int k )
|
|
{
|
|
if ( p == -1 )
|
|
{
|
|
FillBuffer();
|
|
}
|
|
if ( k == 0 )
|
|
{
|
|
return null;
|
|
}
|
|
if ( k < 0 )
|
|
{
|
|
return LB( -k );
|
|
}
|
|
//System.out.print("LT(p="+p+","+k+")=");
|
|
if ( ( p + k - 1 ) >= tokens.Count )
|
|
{
|
|
return tokens[tokens.Count - 1];
|
|
}
|
|
//System.out.println(tokens.get(p+k-1));
|
|
int i = p;
|
|
int n = 1;
|
|
// find k good tokens
|
|
while ( n < k )
|
|
{
|
|
// skip off-channel tokens
|
|
i = SkipOffTokenChannels( i + 1 ); // leave p on valid token
|
|
n++;
|
|
}
|
|
if ( i >= tokens.Count )
|
|
{
|
|
return tokens[tokens.Count - 1];
|
|
}
|
|
|
|
if (i > Range)
|
|
Range = i;
|
|
|
|
return (IToken)tokens[i];
|
|
}
|
|
|
|
/** <summary>Look backwards k tokens on-channel tokens</summary> */
|
|
protected virtual IToken LB( int k )
|
|
{
|
|
//System.out.print("LB(p="+p+","+k+") ");
|
|
if ( p == -1 )
|
|
{
|
|
FillBuffer();
|
|
}
|
|
if ( k == 0 )
|
|
{
|
|
return null;
|
|
}
|
|
if ( ( p - k ) < 0 )
|
|
{
|
|
return null;
|
|
}
|
|
|
|
int i = p;
|
|
int n = 1;
|
|
// find k good tokens looking backwards
|
|
while ( n <= k )
|
|
{
|
|
// skip off-channel tokens
|
|
i = SkipOffTokenChannelsReverse( i - 1 ); // leave p on valid token
|
|
n++;
|
|
}
|
|
if ( i < 0 )
|
|
{
|
|
return null;
|
|
}
|
|
return (IToken)tokens[i];
|
|
}
|
|
|
|
/** <summary>
|
|
* Return absolute token i; ignore which channel the tokens are on;
|
|
* that is, count all tokens not just on-channel tokens.
|
|
* </summary>
|
|
*/
|
|
public virtual IToken Get( int i )
|
|
{
|
|
return (IToken)tokens[i];
|
|
}
|
|
|
|
#if false
|
|
/** Get all tokens from start..stop inclusively */
|
|
public virtual List<IToken> Get(int start, int count)
|
|
{
|
|
if (start < 0)
|
|
throw new ArgumentOutOfRangeException("start");
|
|
if (count < 0)
|
|
throw new ArgumentOutOfRangeException("count");
|
|
|
|
if (p == -1)
|
|
FillBuffer();
|
|
|
|
return new List<IToken>(tokens.Skip(start).Take(count));
|
|
}
|
|
#endif
|
|
|
|
public virtual int LA( int i )
|
|
{
|
|
return LT( i ).Type;
|
|
}
|
|
|
|
public virtual int Mark()
|
|
{
|
|
if ( p == -1 )
|
|
{
|
|
FillBuffer();
|
|
}
|
|
lastMarker = Index;
|
|
return lastMarker;
|
|
}
|
|
|
|
public virtual void Release( int marker )
|
|
{
|
|
// no resources to release
|
|
}
|
|
|
|
public virtual int Count
|
|
{
|
|
get
|
|
{
|
|
return tokens.Count;
|
|
}
|
|
}
|
|
|
|
public virtual void Rewind( int marker )
|
|
{
|
|
Seek( marker );
|
|
}
|
|
|
|
public virtual void Rewind()
|
|
{
|
|
Seek( lastMarker );
|
|
}
|
|
|
|
public virtual void Reset()
|
|
{
|
|
p = 0;
|
|
lastMarker = 0;
|
|
}
|
|
|
|
public virtual void Seek( int index )
|
|
{
|
|
p = index;
|
|
}
|
|
|
|
public virtual ITokenSource TokenSource
|
|
{
|
|
get
|
|
{
|
|
return _tokenSource;
|
|
}
|
|
}
|
|
|
|
public virtual string SourceName
|
|
{
|
|
get
|
|
{
|
|
return TokenSource.SourceName;
|
|
}
|
|
}
|
|
|
|
public override string ToString()
|
|
{
|
|
if ( p == -1 )
|
|
{
|
|
throw new InvalidOperationException( "Buffer is not yet filled." );
|
|
}
|
|
return ToString( 0, tokens.Count - 1 );
|
|
}
|
|
|
|
public virtual string ToString( int start, int stop )
|
|
{
|
|
if ( start < 0 || stop < 0 )
|
|
{
|
|
return null;
|
|
}
|
|
if ( p == -1 )
|
|
{
|
|
throw new InvalidOperationException( "Buffer is not yet filled." );
|
|
}
|
|
if ( stop >= tokens.Count )
|
|
{
|
|
stop = tokens.Count - 1;
|
|
}
|
|
StringBuilder buf = new StringBuilder();
|
|
for ( int i = start; i <= stop; i++ )
|
|
{
|
|
IToken t = tokens[i];
|
|
buf.Append( t.Text );
|
|
}
|
|
return buf.ToString();
|
|
}
|
|
|
|
public virtual string ToString( IToken start, IToken stop )
|
|
{
|
|
if ( start != null && stop != null )
|
|
{
|
|
return ToString( start.TokenIndex, stop.TokenIndex );
|
|
}
|
|
return null;
|
|
}
|
|
}
|
|
}
|