You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

616 lines
26 KiB

/*
* [The "BSD licence"]
* Copyright (c) 2005-2008 Terence Parr
* All rights reserved.
*
* Conversion to C#:
* Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
namespace Antlr.Runtime {
using System.Collections.Generic;
using ArgumentException = System.ArgumentException;
using DebuggerDisplay = System.Diagnostics.DebuggerDisplayAttribute;
using Exception = System.Exception;
using StringBuilder = System.Text.StringBuilder;
using Type = System.Type;
/** Useful for dumping out the input stream after doing some
* augmentation or other manipulations.
*
* You can insert stuff, replace, and delete chunks. Note that the
* operations are done lazily--only if you convert the buffer to a
* String. This is very efficient because you are not moving data around
* all the time. As the buffer of tokens is converted to strings, the
* toString() method(s) check to see if there is an operation at the
* current index. If so, the operation is done and then normal String
* rendering continues on the buffer. This is like having multiple Turing
* machine instruction streams (programs) operating on a single input tape. :)
*
* Since the operations are done lazily at toString-time, operations do not
* screw up the token index values. That is, an insert operation at token
* index i does not change the index values for tokens i+1..n-1.
*
* Because operations never actually alter the buffer, you may always get
* the original token stream back without undoing anything. Since
* the instructions are queued up, you can easily simulate transactions and
* roll back any changes if there is an error just by removing instructions.
* For example,
*
* CharStream input = new ANTLRFileStream("input");
* TLexer lex = new TLexer(input);
* TokenRewriteStream tokens = new TokenRewriteStream(lex);
* T parser = new T(tokens);
* parser.startRule();
*
* Then in the rules, you can execute
* Token t,u;
* ...
* input.insertAfter(t, "text to put after t");}
* input.insertAfter(u, "text after u");}
* System.out.println(tokens.toString());
*
* Actually, you have to cast the 'input' to a TokenRewriteStream. :(
*
* You can also have multiple "instruction streams" and get multiple
* rewrites from a single pass over the input. Just name the instruction
* streams and use that name again when printing the buffer. This could be
* useful for generating a C file and also its header file--all from the
* same buffer:
*
* tokens.insertAfter("pass1", t, "text to put after t");}
* tokens.insertAfter("pass2", u, "text after u");}
* System.out.println(tokens.toString("pass1"));
* System.out.println(tokens.toString("pass2"));
*
* If you don't use named rewrite streams, a "default" stream is used as
* the first example shows.
*/
[System.Serializable]
[DebuggerDisplay("TODO: TokenRewriteStream debugger display")]
public class TokenRewriteStream : CommonTokenStream {
public const string DEFAULT_PROGRAM_NAME = "default";
public const int PROGRAM_INIT_SIZE = 100;
public const int MIN_TOKEN_INDEX = 0;
// Define the rewrite operation hierarchy
protected class RewriteOperation {
/** <summary>What index into rewrites List are we?</summary> */
public int instructionIndex;
/** <summary>Token buffer index.</summary> */
public int index;
public object text;
// outer
protected TokenRewriteStream stream;
protected RewriteOperation(TokenRewriteStream stream, int index, object text) {
this.index = index;
this.text = text;
this.stream = stream;
}
/** <summary>
* Execute the rewrite operation by possibly adding to the buffer.
* Return the index of the next token to operate on.
* </summary>
*/
public virtual int Execute(StringBuilder buf) {
return index;
}
public override string ToString() {
string opName = this.GetType().Name;
int index = opName.IndexOf('$');
opName = opName.Substring(index + 1);
return "<" + opName + "@" + this.index + ":\"" + text + "\">";
}
}
class InsertBeforeOp : RewriteOperation {
public InsertBeforeOp(TokenRewriteStream stream, int index, object text) :
base(stream, index, text) {
}
public override int Execute(StringBuilder buf) {
buf.Append(text);
if (stream._tokens[index].Type != CharStreamConstants.EndOfFile)
buf.Append(stream._tokens[index].Text);
return index + 1;
}
}
/** <summary>
* I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
* instructions.
* </summary>
*/
class ReplaceOp : RewriteOperation {
public int lastIndex;
public ReplaceOp(TokenRewriteStream stream, int from, int to, object text)
: base(stream, from, text) {
lastIndex = to;
}
public override int Execute(StringBuilder buf) {
if (text != null) {
buf.Append(text);
}
return lastIndex + 1;
}
public override string ToString() {
return "<ReplaceOp@" + index + ".." + lastIndex + ":\"" + text + "\">";
}
}
class DeleteOp : ReplaceOp {
public DeleteOp(TokenRewriteStream stream, int from, int to) :
base(stream, from, to, null) {
}
public override string ToString() {
return "<DeleteOp@" + index + ".." + lastIndex + ">";
}
}
/** <summary>
* You may have multiple, named streams of rewrite operations.
* I'm calling these things "programs."
* Maps String (name) -> rewrite (List)
* </summary>
*/
protected IDictionary<string, IList<RewriteOperation>> programs = null;
/** <summary>Map String (program name) -> Integer index</summary> */
protected IDictionary<string, int> lastRewriteTokenIndexes = null;
public TokenRewriteStream() {
Init();
}
protected void Init() {
programs = new Dictionary<string, IList<RewriteOperation>>();
programs[DEFAULT_PROGRAM_NAME] = new List<RewriteOperation>(PROGRAM_INIT_SIZE);
lastRewriteTokenIndexes = new Dictionary<string, int>();
}
public TokenRewriteStream(ITokenSource tokenSource)
: base(tokenSource) {
Init();
}
public TokenRewriteStream(ITokenSource tokenSource, int channel)
: base(tokenSource, channel) {
Init();
}
public virtual void Rollback(int instructionIndex) {
Rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
}
/** <summary>
* Rollback the instruction stream for a program so that
* the indicated instruction (via instructionIndex) is no
* longer in the stream. UNTESTED!
* </summary>
*/
public virtual void Rollback(string programName, int instructionIndex) {
IList<RewriteOperation> @is;
if (programs.TryGetValue(programName, out @is) && @is != null) {
List<RewriteOperation> sublist = new List<RewriteOperation>();
for (int i = MIN_TOKEN_INDEX; i <= instructionIndex; i++)
sublist.Add(@is[i]);
programs[programName] = sublist;
}
}
public virtual void DeleteProgram() {
DeleteProgram(DEFAULT_PROGRAM_NAME);
}
/** <summary>Reset the program so that no instructions exist</summary> */
public virtual void DeleteProgram(string programName) {
Rollback(programName, MIN_TOKEN_INDEX);
}
public virtual void InsertAfter(IToken t, object text) {
InsertAfter(DEFAULT_PROGRAM_NAME, t, text);
}
public virtual void InsertAfter(int index, object text) {
InsertAfter(DEFAULT_PROGRAM_NAME, index, text);
}
public virtual void InsertAfter(string programName, IToken t, object text) {
InsertAfter(programName, t.TokenIndex, text);
}
public virtual void InsertAfter(string programName, int index, object text) {
// to insert after, just insert before next index (even if past end)
InsertBefore(programName, index + 1, text);
//addToSortedRewriteList(programName, new InsertAfterOp(index,text));
}
public virtual void InsertBefore(IToken t, object text) {
InsertBefore(DEFAULT_PROGRAM_NAME, t, text);
}
public virtual void InsertBefore(int index, object text) {
InsertBefore(DEFAULT_PROGRAM_NAME, index, text);
}
public virtual void InsertBefore(string programName, IToken t, object text) {
InsertBefore(programName, t.TokenIndex, text);
}
public virtual void InsertBefore(string programName, int index, object text) {
//addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
RewriteOperation op = new InsertBeforeOp(this, index, text);
IList<RewriteOperation> rewrites = GetProgram(programName);
op.instructionIndex = rewrites.Count;
rewrites.Add(op);
}
public virtual void Replace(int index, object text) {
Replace(DEFAULT_PROGRAM_NAME, index, index, text);
}
public virtual void Replace(int from, int to, object text) {
Replace(DEFAULT_PROGRAM_NAME, from, to, text);
}
public virtual void Replace(IToken indexT, object text) {
Replace(DEFAULT_PROGRAM_NAME, indexT, indexT, text);
}
public virtual void Replace(IToken from, IToken to, object text) {
Replace(DEFAULT_PROGRAM_NAME, from, to, text);
}
public virtual void Replace(string programName, int from, int to, object text) {
if (from > to || from < 0 || to < 0 || to >= _tokens.Count) {
throw new ArgumentException("replace: range invalid: " + from + ".." + to + "(size=" + _tokens.Count + ")");
}
RewriteOperation op = new ReplaceOp(this, from, to, text);
IList<RewriteOperation> rewrites = GetProgram(programName);
op.instructionIndex = rewrites.Count;
rewrites.Add(op);
}
public virtual void Replace(string programName, IToken from, IToken to, object text) {
Replace(programName,
from.TokenIndex,
to.TokenIndex,
text);
}
public virtual void Delete(int index) {
Delete(DEFAULT_PROGRAM_NAME, index, index);
}
public virtual void Delete(int from, int to) {
Delete(DEFAULT_PROGRAM_NAME, from, to);
}
public virtual void Delete(IToken indexT) {
Delete(DEFAULT_PROGRAM_NAME, indexT, indexT);
}
public virtual void Delete(IToken from, IToken to) {
Delete(DEFAULT_PROGRAM_NAME, from, to);
}
public virtual void Delete(string programName, int from, int to) {
Replace(programName, from, to, null);
}
public virtual void Delete(string programName, IToken from, IToken to) {
Replace(programName, from, to, null);
}
public virtual int GetLastRewriteTokenIndex() {
return GetLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
}
protected virtual int GetLastRewriteTokenIndex(string programName) {
int value;
if (lastRewriteTokenIndexes.TryGetValue(programName, out value))
return value;
return -1;
}
protected virtual void SetLastRewriteTokenIndex(string programName, int i) {
lastRewriteTokenIndexes[programName] = i;
}
protected virtual IList<RewriteOperation> GetProgram(string name) {
IList<RewriteOperation> @is;
if (!programs.TryGetValue(name, out @is) || @is == null) {
@is = InitializeProgram(name);
}
return @is;
}
private IList<RewriteOperation> InitializeProgram(string name) {
IList<RewriteOperation> @is = new List<RewriteOperation>(PROGRAM_INIT_SIZE);
programs[name] = @is;
return @is;
}
public virtual string ToOriginalString() {
Fill();
return ToOriginalString(MIN_TOKEN_INDEX, Count - 1);
}
public virtual string ToOriginalString(int start, int end) {
StringBuilder buf = new StringBuilder();
for (int i = start; i >= MIN_TOKEN_INDEX && i <= end && i < _tokens.Count; i++) {
if (Get(i).Type != CharStreamConstants.EndOfFile)
buf.Append(Get(i).Text);
}
return buf.ToString();
}
public override string ToString() {
Fill();
return ToString(MIN_TOKEN_INDEX, Count - 1);
}
public virtual string ToString(string programName) {
Fill();
return ToString(programName, MIN_TOKEN_INDEX, Count - 1);
}
public override string ToString(int start, int end) {
return ToString(DEFAULT_PROGRAM_NAME, start, end);
}
public virtual string ToString(string programName, int start, int end) {
IList<RewriteOperation> rewrites;
if (!programs.TryGetValue(programName, out rewrites))
rewrites = null;
// ensure start/end are in range
if (end > _tokens.Count - 1)
end = _tokens.Count - 1;
if (start < 0)
start = 0;
if (rewrites == null || rewrites.Count == 0) {
return ToOriginalString(start, end); // no instructions to execute
}
StringBuilder buf = new StringBuilder();
// First, optimize instruction stream
IDictionary<int, RewriteOperation> indexToOp = ReduceToSingleOperationPerIndex(rewrites);
// Walk buffer, executing instructions and emitting tokens
int i = start;
while (i <= end && i < _tokens.Count) {
RewriteOperation op;
bool exists = indexToOp.TryGetValue(i, out op);
if (exists) {
// remove so any left have index size-1
indexToOp.Remove(i);
}
if (!exists || op == null) {
IToken t = _tokens[i];
// no operation at that index, just dump token
if (t.Type != CharStreamConstants.EndOfFile)
buf.Append(t.Text);
i++; // move to next token
} else {
i = op.Execute(buf); // execute operation and skip
}
}
// include stuff after end if it's last index in buffer
// So, if they did an insertAfter(lastValidIndex, "foo"), include
// foo if end==lastValidIndex.
if (end == _tokens.Count - 1) {
// Scan any remaining operations after last token
// should be included (they will be inserts).
foreach (RewriteOperation op in indexToOp.Values) {
if (op.index >= _tokens.Count - 1)
buf.Append(op.text);
}
}
return buf.ToString();
}
/** We need to combine operations and report invalid operations (like
* overlapping replaces that are not completed nested). Inserts to
* same index need to be combined etc... Here are the cases:
*
* I.i.u I.j.v leave alone, nonoverlapping
* I.i.u I.i.v combine: Iivu
*
* R.i-j.u R.x-y.v | i-j in x-y delete first R
* R.i-j.u R.i-j.v delete first R
* R.i-j.u R.x-y.v | x-y in i-j ERROR
* R.i-j.u R.x-y.v | boundaries overlap ERROR
*
* I.i.u R.x-y.v | i in x-y delete I
* I.i.u R.x-y.v | i not in x-y leave alone, nonoverlapping
* R.x-y.v I.i.u | i in x-y ERROR
* R.x-y.v I.x.u R.x-y.uv (combine, delete I)
* R.x-y.v I.i.u | i not in x-y leave alone, nonoverlapping
*
* I.i.u = insert u before op @ index i
* R.x-y.u = replace x-y indexed tokens with u
*
* First we need to examine replaces. For any replace op:
*
* 1. wipe out any insertions before op within that range.
* 2. Drop any replace op before that is contained completely within
* that range.
* 3. Throw exception upon boundary overlap with any previous replace.
*
* Then we can deal with inserts:
*
* 1. for any inserts to same index, combine even if not adjacent.
* 2. for any prior replace with same left boundary, combine this
* insert with replace and delete this replace.
* 3. throw exception if index in same range as previous replace
*
* Don't actually delete; make op null in list. Easier to walk list.
* Later we can throw as we add to index -> op map.
*
* Note that I.2 R.2-2 will wipe out I.2 even though, technically, the
* inserted stuff would be before the replace range. But, if you
* add tokens in front of a method body '{' and then delete the method
* body, I think the stuff before the '{' you added should disappear too.
*
* Return a map from token index to operation.
*/
protected virtual IDictionary<int, RewriteOperation> ReduceToSingleOperationPerIndex(IList<RewriteOperation> rewrites) {
//System.out.println("rewrites="+rewrites);
// WALK REPLACES
for (int i = 0; i < rewrites.Count; i++) {
RewriteOperation op = rewrites[i];
if (op == null)
continue;
if (!(op is ReplaceOp))
continue;
ReplaceOp rop = (ReplaceOp)rewrites[i];
// Wipe prior inserts within range
var inserts = GetKindOfOps(rewrites, typeof(InsertBeforeOp), i);
for (int j = 0; j < inserts.Count; j++) {
InsertBeforeOp iop = (InsertBeforeOp)inserts[j];
if (iop.index >= rop.index && iop.index <= rop.lastIndex) {
// delete insert as it's a no-op.
rewrites[iop.instructionIndex] = null;
}
}
// Drop any prior replaces contained within
var prevReplaces = GetKindOfOps(rewrites, typeof(ReplaceOp), i);
for (int j = 0; j < prevReplaces.Count; j++) {
ReplaceOp prevRop = (ReplaceOp)prevReplaces[j];
if (prevRop.index >= rop.index && prevRop.lastIndex <= rop.lastIndex) {
// delete replace as it's a no-op.
rewrites[prevRop.instructionIndex] = null;
continue;
}
// throw exception unless disjoint or identical
bool disjoint =
prevRop.lastIndex < rop.index || prevRop.index > rop.lastIndex;
bool same =
prevRop.index == rop.index && prevRop.lastIndex == rop.lastIndex;
if (!disjoint && !same) {
throw new ArgumentException("replace op boundaries of " + rop +
" overlap with previous " + prevRop);
}
}
}
// WALK INSERTS
for (int i = 0; i < rewrites.Count; i++) {
RewriteOperation op = (RewriteOperation)rewrites[i];
if (op == null)
continue;
if (!(op is InsertBeforeOp))
continue;
InsertBeforeOp iop = (InsertBeforeOp)rewrites[i];
// combine current insert with prior if any at same index
var prevInserts = GetKindOfOps(rewrites, typeof(InsertBeforeOp), i);
for (int j = 0; j < prevInserts.Count; j++) {
InsertBeforeOp prevIop = (InsertBeforeOp)prevInserts[j];
if (prevIop.index == iop.index) { // combine objects
// convert to strings...we're in process of toString'ing
// whole token buffer so no lazy eval issue with any templates
iop.text = CatOpText(iop.text, prevIop.text);
// delete redundant prior insert
rewrites[prevIop.instructionIndex] = null;
}
}
// look for replaces where iop.index is in range; error
var prevReplaces = GetKindOfOps(rewrites, typeof(ReplaceOp), i);
for (int j = 0; j < prevReplaces.Count; j++) {
ReplaceOp rop = (ReplaceOp)prevReplaces[j];
if (iop.index == rop.index) {
rop.text = CatOpText(iop.text, rop.text);
rewrites[i] = null; // delete current insert
continue;
}
if (iop.index >= rop.index && iop.index <= rop.lastIndex) {
throw new ArgumentException("insert op " + iop +
" within boundaries of previous " + rop);
}
}
}
// System.out.println("rewrites after="+rewrites);
IDictionary<int, RewriteOperation> m = new Dictionary<int, RewriteOperation>();
for (int i = 0; i < rewrites.Count; i++) {
RewriteOperation op = (RewriteOperation)rewrites[i];
if (op == null)
continue; // ignore deleted ops
RewriteOperation existing;
if (m.TryGetValue(op.index, out existing) && existing != null) {
throw new Exception("should only be one op per index");
}
m[op.index] = op;
}
//System.out.println("index to op: "+m);
return m;
}
protected virtual string CatOpText(object a, object b) {
return string.Concat(a, b);
}
protected virtual IList<RewriteOperation> GetKindOfOps(IList<RewriteOperation> rewrites, Type kind) {
return GetKindOfOps(rewrites, kind, rewrites.Count);
}
/** <summary>Get all operations before an index of a particular kind</summary> */
protected virtual IList<RewriteOperation> GetKindOfOps(IList<RewriteOperation> rewrites, Type kind, int before) {
IList<RewriteOperation> ops = new List<RewriteOperation>();
for (int i = 0; i < before && i < rewrites.Count; i++) {
RewriteOperation op = rewrites[i];
if (op == null)
continue; // ignore deleted
if (op.GetType() == kind)
ops.Add(op);
}
return ops;
}
public virtual string ToDebugString() {
return ToDebugString(MIN_TOKEN_INDEX, Count - 1);
}
public virtual string ToDebugString(int start, int end) {
StringBuilder buf = new StringBuilder();
for (int i = start; i >= MIN_TOKEN_INDEX && i <= end && i < _tokens.Count; i++) {
buf.Append(Get(i));
}
return buf.ToString();
}
}
}