You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
373 lines
12 KiB
373 lines
12 KiB
/*
|
|
* [The "BSD licence"]
|
|
* Copyright (c) 2005-2008 Terence Parr
|
|
* All rights reserved.
|
|
*
|
|
* Conversion to C#:
|
|
* Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* 3. The name of the author may not be used to endorse or promote products
|
|
* derived from this software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
namespace Antlr.Runtime {
|
|
using System.Collections.Generic;
|
|
using CLSCompliant = System.CLSCompliantAttribute;
|
|
using IndexOutOfRangeException = System.IndexOutOfRangeException;
|
|
using StringBuilder = System.Text.StringBuilder;
|
|
|
|
/** Buffer all input tokens but do on-demand fetching of new tokens from
|
|
* lexer. Useful when the parser or lexer has to set context/mode info before
|
|
* proper lexing of future tokens. The ST template parser needs this,
|
|
* for example, because it has to constantly flip back and forth between
|
|
* inside/output templates. E.g., <names:{hi, <it>}> has to parse names
|
|
* as part of an expression but "hi, <it>" as a nested template.
|
|
*
|
|
* You can't use this stream if you pass whitespace or other off-channel
|
|
* tokens to the parser. The stream can't ignore off-channel tokens.
|
|
* (UnbufferedTokenStream is the same way.)
|
|
*
|
|
* This is not a subclass of UnbufferedTokenStream because I don't want
|
|
* to confuse small moving window of tokens it uses for the full buffer.
|
|
*/
|
|
[System.Serializable]
|
|
public class BufferedTokenStream : ITokenStream, ITokenStreamInformation {
|
|
private ITokenSource _tokenSource;
|
|
|
|
/** Record every single token pulled from the source so we can reproduce
|
|
* chunks of it later. The buffer in LookaheadStream overlaps sometimes
|
|
* as its moving window moves through the input. This list captures
|
|
* everything so we can access complete input text.
|
|
*/
|
|
[CLSCompliant(false)]
|
|
protected List<IToken> _tokens = new List<IToken>(100);
|
|
|
|
/** Track the last mark() call result value for use in rewind(). */
|
|
private int _lastMarker;
|
|
|
|
/** The index into the tokens list of the current token (next token
|
|
* to consume). tokens[p] should be LT(1). p=-1 indicates need
|
|
* to initialize with first token. The ctor doesn't get a token.
|
|
* First call to LT(1) or whatever gets the first token and sets p=0;
|
|
*/
|
|
[CLSCompliant(false)]
|
|
protected int _p = -1;
|
|
|
|
public BufferedTokenStream() {
|
|
}
|
|
|
|
public BufferedTokenStream(ITokenSource tokenSource) {
|
|
this._tokenSource = tokenSource;
|
|
}
|
|
|
|
public virtual ITokenSource TokenSource {
|
|
get {
|
|
return _tokenSource;
|
|
}
|
|
set {
|
|
this._tokenSource = value;
|
|
_tokens.Clear();
|
|
_p = -1;
|
|
}
|
|
}
|
|
|
|
public virtual int Index {
|
|
get {
|
|
return _p;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// How deep have we gone?
|
|
/// </summary>
|
|
public virtual int Range {
|
|
get;
|
|
protected set;
|
|
}
|
|
|
|
public virtual int Count {
|
|
get {
|
|
return _tokens.Count;
|
|
}
|
|
}
|
|
|
|
public virtual string SourceName {
|
|
get {
|
|
return _tokenSource.SourceName;
|
|
}
|
|
}
|
|
|
|
public virtual IToken LastToken {
|
|
get {
|
|
return LB(1);
|
|
}
|
|
}
|
|
|
|
public virtual IToken LastRealToken {
|
|
get {
|
|
int i = 0;
|
|
IToken token;
|
|
do {
|
|
i++;
|
|
token = LB(i);
|
|
} while (token != null && token.Line <= 0);
|
|
|
|
return token;
|
|
}
|
|
}
|
|
|
|
public virtual int MaxLookBehind {
|
|
get {
|
|
return int.MaxValue;
|
|
}
|
|
}
|
|
|
|
public virtual int Mark() {
|
|
if (_p == -1)
|
|
Setup();
|
|
_lastMarker = Index;
|
|
return _lastMarker;
|
|
}
|
|
|
|
public virtual void Release(int marker) {
|
|
// no resources to release
|
|
}
|
|
|
|
public virtual void Rewind(int marker) {
|
|
Seek(marker);
|
|
}
|
|
|
|
public virtual void Rewind() {
|
|
Seek(_lastMarker);
|
|
}
|
|
|
|
public virtual void Reset() {
|
|
_p = 0;
|
|
_lastMarker = 0;
|
|
}
|
|
|
|
public virtual void Seek(int index) {
|
|
_p = index;
|
|
}
|
|
|
|
/** Move the input pointer to the next incoming token. The stream
|
|
* must become active with LT(1) available. consume() simply
|
|
* moves the input pointer so that LT(1) points at the next
|
|
* input symbol. Consume at least one token.
|
|
*
|
|
* Walk past any token not on the channel the parser is listening to.
|
|
*/
|
|
public virtual void Consume() {
|
|
if (_p == -1)
|
|
Setup();
|
|
_p++;
|
|
Sync(_p);
|
|
}
|
|
|
|
/** Make sure index i in tokens has a token. */
|
|
protected virtual void Sync(int i) {
|
|
int n = i - _tokens.Count + 1; // how many more elements we need?
|
|
if (n > 0)
|
|
Fetch(n);
|
|
}
|
|
|
|
/** add n elements to buffer */
|
|
protected virtual void Fetch(int n) {
|
|
for (int i = 0; i < n; i++) {
|
|
IToken t = TokenSource.NextToken();
|
|
t.TokenIndex = _tokens.Count;
|
|
_tokens.Add(t);
|
|
if (t.Type == CharStreamConstants.EndOfFile)
|
|
break;
|
|
}
|
|
}
|
|
|
|
public virtual IToken Get(int i) {
|
|
if (i < 0 || i >= _tokens.Count) {
|
|
throw new IndexOutOfRangeException("token index " + i + " out of range 0.." + (_tokens.Count - 1));
|
|
}
|
|
return _tokens[i];
|
|
}
|
|
|
|
#if false // why is this different from GetTokens(start, count) ?
|
|
/// <summary>
|
|
/// Get all tokens from start..(start+count-1) inclusively
|
|
/// </summary>
|
|
public virtual List<IToken> Get(int start, int count)
|
|
{
|
|
if (start < 0)
|
|
throw new ArgumentOutOfRangeException("start");
|
|
if (count < 0)
|
|
throw new ArgumentOutOfRangeException("count");
|
|
if (start + count >= _tokens.Count)
|
|
throw new ArgumentException();
|
|
|
|
if (_p == -1)
|
|
Setup();
|
|
|
|
List<IToken> subset = new List<IToken>(count);
|
|
for (int i = 0; i < count; i++)
|
|
{
|
|
IToken token = _tokens[i];
|
|
if (token.Type == TokenTypes.EndOfFile)
|
|
break;
|
|
|
|
subset.Add(token);
|
|
}
|
|
|
|
return subset;
|
|
}
|
|
#endif
|
|
|
|
public virtual int LA(int i) {
|
|
return LT(i).Type;
|
|
}
|
|
|
|
protected virtual IToken LB(int k) {
|
|
if ((_p - k) < 0)
|
|
return null;
|
|
|
|
return _tokens[_p - k];
|
|
}
|
|
|
|
public virtual IToken LT(int k) {
|
|
if (_p == -1)
|
|
Setup();
|
|
if (k == 0)
|
|
return null;
|
|
if (k < 0)
|
|
return LB(-k);
|
|
|
|
int i = _p + k - 1;
|
|
Sync(i);
|
|
if (i >= _tokens.Count) {
|
|
// EOF must be last token
|
|
return _tokens[_tokens.Count - 1];
|
|
}
|
|
|
|
if (i > Range)
|
|
Range = i;
|
|
|
|
return _tokens[_p + k - 1];
|
|
}
|
|
|
|
protected virtual void Setup() {
|
|
Sync(0);
|
|
_p = 0;
|
|
}
|
|
|
|
public virtual List<IToken> GetTokens() {
|
|
return _tokens;
|
|
}
|
|
|
|
public virtual List<IToken> GetTokens(int start, int stop) {
|
|
return GetTokens(start, stop, default(BitSet));
|
|
}
|
|
|
|
/** Given a start and stop index, return a List of all tokens in
|
|
* the token type BitSet. Return null if no tokens were found. This
|
|
* method looks at both on and off channel tokens.
|
|
*/
|
|
public virtual List<IToken> GetTokens(int start, int stop, BitSet types) {
|
|
if (_p == -1)
|
|
Setup();
|
|
if (stop >= _tokens.Count)
|
|
stop = _tokens.Count - 1;
|
|
if (start < 0)
|
|
start = 0;
|
|
if (start > stop)
|
|
return null;
|
|
|
|
// list = tokens[start:stop]:{Token t, t.getType() in types}
|
|
List<IToken> filteredTokens = new List<IToken>();
|
|
for (int i = start; i <= stop; i++) {
|
|
IToken t = _tokens[i];
|
|
if (types == null || types.Member(t.Type)) {
|
|
filteredTokens.Add(t);
|
|
}
|
|
}
|
|
if (filteredTokens.Count == 0) {
|
|
filteredTokens = null;
|
|
}
|
|
return filteredTokens;
|
|
}
|
|
|
|
public virtual List<IToken> GetTokens(int start, int stop, IEnumerable<int> types) {
|
|
return GetTokens(start, stop, new BitSet(types));
|
|
}
|
|
|
|
public virtual List<IToken> GetTokens(int start, int stop, int ttype) {
|
|
return GetTokens(start, stop, BitSet.Of(ttype));
|
|
}
|
|
|
|
public override string ToString() {
|
|
if (_p == -1)
|
|
Setup();
|
|
|
|
Fill();
|
|
return ToString(0, _tokens.Count - 1);
|
|
}
|
|
|
|
public virtual string ToString(int start, int stop) {
|
|
if (start < 0 || stop < 0)
|
|
return null;
|
|
if (_p == -1)
|
|
Setup();
|
|
if (stop >= _tokens.Count)
|
|
stop = _tokens.Count - 1;
|
|
|
|
StringBuilder buf = new StringBuilder();
|
|
for (int i = start; i <= stop; i++) {
|
|
IToken t = _tokens[i];
|
|
if (t.Type == CharStreamConstants.EndOfFile)
|
|
break;
|
|
buf.Append(t.Text);
|
|
}
|
|
|
|
return buf.ToString();
|
|
}
|
|
|
|
public virtual string ToString(IToken start, IToken stop) {
|
|
if (start != null && stop != null) {
|
|
return ToString(start.TokenIndex, stop.TokenIndex);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
public virtual void Fill() {
|
|
if (_p == -1)
|
|
Setup();
|
|
|
|
if (_tokens[_p].Type == CharStreamConstants.EndOfFile)
|
|
return;
|
|
|
|
int i = _p + 1;
|
|
Sync(i);
|
|
while (_tokens[i].Type != CharStreamConstants.EndOfFile) {
|
|
i++;
|
|
Sync(i);
|
|
}
|
|
}
|
|
}
|
|
}
|