You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
393 lines
9.8 KiB
393 lines
9.8 KiB
// [The "BSD licence"]
|
|
// Copyright (c) 2010 Alan Condit
|
|
// All rights reserved.
|
|
//
|
|
// Redistribution and use in source and binary forms, with or without
|
|
// modification, are permitted provided that the following conditions
|
|
// are met:
|
|
// 1. Redistributions of source code must retain the above copyright
|
|
// notice, this list of conditions and the following disclaimer.
|
|
// 2. Redistributions in binary form must reproduce the above copyright
|
|
// notice, this list of conditions and the following disclaimer in the
|
|
// documentation and/or other materials provided with the distribution.
|
|
// 3. The name of the author may not be used to endorse or promote products
|
|
// derived from this software without specific prior written permission.
|
|
//
|
|
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
#import "BufferedTokenStream.h"
|
|
#import "TokenSource.h"
|
|
#import "CommonTreeAdaptor.h"
|
|
#import "RuntimeException.h"
|
|
|
|
extern NSInteger debug;
|
|
|
|
@implementation BufferedTokenStream
|
|
|
|
@synthesize tokenSource;
|
|
@synthesize tokens;
|
|
@synthesize lastMarker;
|
|
@synthesize index;
|
|
@synthesize range;
|
|
|
|
+ (BufferedTokenStream *) newBufferedTokenStream
|
|
{
|
|
return [[BufferedTokenStream alloc] init];
|
|
}
|
|
|
|
+ (BufferedTokenStream *) newBufferedTokenStreamWith:(id<TokenSource>)aSource
|
|
{
|
|
return [[BufferedTokenStream alloc] initWithTokenSource:aSource];
|
|
}
|
|
|
|
- (BufferedTokenStream *) init
|
|
{
|
|
if ((self = [super init]) != nil)
|
|
{
|
|
tokenSource = nil;
|
|
tokens = [[AMutableArray arrayWithCapacity:1000] retain];
|
|
index = -1;
|
|
range = -1;
|
|
}
|
|
return self;
|
|
}
|
|
|
|
-(id) initWithTokenSource:(id<TokenSource>)aSource
|
|
{
|
|
if ((self = [super init]) != nil)
|
|
{
|
|
tokenSource = [aSource retain];
|
|
tokens = [[AMutableArray arrayWithCapacity:1000] retain];
|
|
index = -1;
|
|
range = -1;
|
|
}
|
|
return self;
|
|
}
|
|
|
|
- (id) copyWithZone:(NSZone *)aZone
|
|
{
|
|
BufferedTokenStream *copy;
|
|
|
|
copy = [[[self class] allocWithZone:aZone] init];
|
|
copy.tokenSource = self.tokenSource;
|
|
if ( self.tokens )
|
|
copy.tokens = [tokens copyWithZone:aZone];
|
|
copy.lastMarker = self.lastMarker;
|
|
copy.index = self.index;
|
|
copy.range = self.range;
|
|
return copy;
|
|
}
|
|
|
|
- (void)dealloc
|
|
{
|
|
#ifdef DEBUG_DEALLOC
|
|
NSLog( @"called dealloc in BufferedTokenStream" );
|
|
#endif
|
|
if ( tokens ) [tokens release];
|
|
if ( tokenSource ) [tokenSource release];
|
|
[super dealloc];
|
|
}
|
|
|
|
- (NSUInteger)line
|
|
{
|
|
return ((CommonToken *)[tokens objectAtIndex:index]).line;
|
|
}
|
|
|
|
- (NSUInteger)charPositionInLine
|
|
{
|
|
return ((CommonToken *)[tokens objectAtIndex:index]).charPositionInLine;
|
|
}
|
|
|
|
- (id<TokenSource>) getTokenSource
|
|
{
|
|
return tokenSource;
|
|
}
|
|
|
|
- (NSInteger) getRange
|
|
{
|
|
return range;
|
|
}
|
|
|
|
- (void) setRange:(NSInteger)anInt
|
|
{
|
|
range = anInt;
|
|
}
|
|
|
|
- (NSInteger) mark
|
|
{
|
|
if ( index == -1 ) {
|
|
[self setup];
|
|
// [self fill];
|
|
}
|
|
lastMarker = self.index;
|
|
return lastMarker;
|
|
}
|
|
|
|
- (void) release:(NSInteger) marker
|
|
{
|
|
// no resources to release
|
|
}
|
|
|
|
- (void) rewind:(NSInteger) marker
|
|
{
|
|
[self seek:marker];
|
|
}
|
|
|
|
- (void) rewind
|
|
{
|
|
[self seek:lastMarker];
|
|
}
|
|
|
|
- (void) reset
|
|
{
|
|
index = 0;
|
|
lastMarker = 0;
|
|
}
|
|
|
|
- (void) seek:(NSInteger) anIndex
|
|
{
|
|
index = anIndex;
|
|
}
|
|
|
|
- (NSInteger) size
|
|
{
|
|
return [tokens count];
|
|
}
|
|
|
|
/** Move the input pointer to the next incoming token. The stream
|
|
* must become active with LT(1) available. consume() simply
|
|
* moves the input pointer so that LT(1) points at the next
|
|
* input symbol. Consume at least one token.
|
|
*
|
|
* Walk past any token not on the channel the parser is listening to.
|
|
*/
|
|
- (void) consume
|
|
{
|
|
if ( index == -1 ) {
|
|
[self setup];
|
|
// [self fill];
|
|
}
|
|
index++;
|
|
[self sync:index];
|
|
}
|
|
|
|
/** Make sure index i in tokens has a token. */
|
|
- (void) sync:(NSInteger) i
|
|
{
|
|
// how many more elements we need?
|
|
NSInteger n = (i - [tokens count]) + 1;
|
|
if (debug > 1) NSLog(@"[self sync:%d] needs %d\n", i, n);
|
|
if ( n > 0 )
|
|
[self fetch:n];
|
|
}
|
|
|
|
/** add n elements to buffer */
|
|
- (void) fetch:(NSInteger)n
|
|
{
|
|
for (NSInteger i=1; i <= n; i++) {
|
|
id<Token> t = [tokenSource nextToken];
|
|
[t setTokenIndex:[tokens count]];
|
|
if (debug > 1) NSLog(@"adding %@ at index %d\n", [t text], [tokens count]);
|
|
[tokens addObject:t];
|
|
if ( t.type == TokenTypeEOF )
|
|
break;
|
|
}
|
|
}
|
|
|
|
- (id<Token>) getToken:(NSInteger) i
|
|
{
|
|
if ( i < 0 || i >= [tokens count] ) {
|
|
@throw [NoSuchElementException newException:[NSString stringWithFormat:@"token index %d out of range 0..%d", i, [tokens count]-1]];
|
|
}
|
|
return [tokens objectAtIndex:i];
|
|
}
|
|
|
|
/** Get all tokens from start..stop inclusively */
|
|
- (AMutableArray *)getFrom:(NSInteger)startIndex To:(NSInteger)stopIndex
|
|
{
|
|
if ( startIndex < 0 || stopIndex < 0 )
|
|
return nil;
|
|
if ( index == -1 ) {
|
|
[self setup];
|
|
// [self fill];
|
|
}
|
|
AMutableArray *subset = [AMutableArray arrayWithCapacity:5];
|
|
if ( stopIndex >= [tokens count] )
|
|
stopIndex = [tokens count]-1;
|
|
for (NSInteger i = startIndex; i <= stopIndex; i++) {
|
|
id<Token>t = [tokens objectAtIndex:i];
|
|
if ( t.type == TokenTypeEOF )
|
|
break;
|
|
[subset addObject:t];
|
|
}
|
|
return subset;
|
|
}
|
|
|
|
- (NSInteger) LA:(NSInteger)i
|
|
{
|
|
return [[self LT:i] type];
|
|
}
|
|
|
|
- (id<Token>) LB:(NSInteger)k
|
|
{
|
|
if ( (index - k) < 0 )
|
|
return nil;
|
|
return [tokens objectAtIndex:(index-k)];
|
|
}
|
|
|
|
- (id<Token>) LT:(NSInteger)k
|
|
{
|
|
if ( index == -1 ) {
|
|
[self setup];
|
|
// [self fill];
|
|
}
|
|
if ( k == 0 )
|
|
return nil;
|
|
if ( k < 0 )
|
|
return [self LB:-k];
|
|
|
|
NSInteger i = index + k - 1;
|
|
[self sync:i];
|
|
if ( i >= [tokens count] ) { // return EOF token
|
|
// EOF must be last token
|
|
return [tokens objectAtIndex:([tokens count]-1)];
|
|
}
|
|
if ( i > range )
|
|
range = i;
|
|
return [tokens objectAtIndex:i];
|
|
}
|
|
|
|
- (void) setup
|
|
{
|
|
[self sync:0];
|
|
index = 0;
|
|
}
|
|
|
|
/** Reset this token stream by setting its token source. */
|
|
- (void) setTokenSource:(id<TokenSource>) aTokenSource
|
|
{
|
|
tokenSource = aTokenSource;
|
|
if ( [tokens count] )
|
|
[tokens removeAllObjects];
|
|
index = -1;
|
|
}
|
|
|
|
- (AMutableArray *)getTokens
|
|
{
|
|
return tokens;
|
|
}
|
|
|
|
- (AMutableArray *)getTokensFrom:(NSInteger) startIndex To:(NSInteger) stopIndex
|
|
{
|
|
return [self getTokensFrom:startIndex To:stopIndex With:(ANTLRBitSet *)nil];
|
|
}
|
|
|
|
/** Given a start and stop index, return a List of all tokens in
|
|
* the token type BitSet. Return null if no tokens were found. This
|
|
* method looks at both on and off channel tokens.
|
|
*/
|
|
- (AMutableArray *)getTokensFrom:(NSInteger)startIndex To:(NSInteger)stopIndex With:(ANTLRBitSet *)types
|
|
{
|
|
if ( index == -1 ) {
|
|
[self setup];
|
|
// [self fill];
|
|
}
|
|
if ( stopIndex >= [tokens count] )
|
|
stopIndex = [tokens count]-1;
|
|
if ( startIndex < 0 )
|
|
startIndex = 0;
|
|
if ( startIndex > stopIndex )
|
|
return nil;
|
|
|
|
// list = tokens[start:stop]:{Token t, t.getType() in types}
|
|
AMutableArray *filteredTokens = [AMutableArray arrayWithCapacity:5];
|
|
for (NSInteger i = startIndex; i <= stopIndex; i++) {
|
|
id<Token>t = [tokens objectAtIndex:i];
|
|
if ( types == nil || [types member:t.type] ) {
|
|
[filteredTokens addObject:t];
|
|
}
|
|
}
|
|
if ( [filteredTokens count] == 0 ) {
|
|
filteredTokens = nil;
|
|
}
|
|
return filteredTokens;
|
|
}
|
|
|
|
- (AMutableArray *)getTokensFrom:(NSInteger)startIndex To:(NSInteger)stopIndex WithType:(NSInteger)ttype
|
|
{
|
|
return [self getTokensFrom:startIndex To:stopIndex With:[ANTLRBitSet of:ttype]];
|
|
}
|
|
|
|
- (AMutableArray *)getTokensFrom:(NSInteger)startIndex To:(NSInteger)stopIndex WithList:(AMutableArray *)types
|
|
{
|
|
return [self getTokensFrom:startIndex To:stopIndex With:[ANTLRBitSet newBitSetWithArray:types]];
|
|
}
|
|
|
|
- (NSString *)getSourceName
|
|
{
|
|
return [tokenSource getSourceName];
|
|
}
|
|
|
|
/** Grab *all* tokens from stream and return string */
|
|
- (NSString *) toString
|
|
{
|
|
if ( index == -1 ) {
|
|
[self setup];
|
|
}
|
|
[self fill];
|
|
return [self toStringFromStart:0 ToEnd:[tokens count]-1];
|
|
}
|
|
|
|
- (NSString *) toStringFromStart:(NSInteger)startIdx ToEnd:(NSInteger)stopIdx
|
|
{
|
|
if ( startIdx < 0 || stopIdx < 0 )
|
|
return nil;
|
|
if ( index == -1 ) {
|
|
[self setup];
|
|
}
|
|
if ( stopIdx >= [tokens count] )
|
|
stopIdx = [tokens count]-1;
|
|
NSMutableString *buf = [NSMutableString stringWithCapacity:5];
|
|
for (NSInteger i = startIdx; i <= stopIdx; i++) {
|
|
id<Token>t = [tokens objectAtIndex:i];
|
|
if ( t.type == TokenTypeEOF )
|
|
break;
|
|
[buf appendString:[t text]];
|
|
}
|
|
return buf;
|
|
}
|
|
|
|
- (NSString *) toStringFromToken:(id<Token>)startToken ToToken:(id<Token>)stopToken
|
|
{
|
|
if ( startToken != nil && stopToken != nil ) {
|
|
return [self toStringFromStart:[startToken getTokenIndex] ToEnd:[stopToken getTokenIndex]];
|
|
}
|
|
return nil;
|
|
}
|
|
|
|
/** Get all tokens from lexer until EOF */
|
|
- (void) fill
|
|
{
|
|
if ( index == -1 ) [self setup];
|
|
if ( [((CommonToken *)[tokens objectAtIndex:index]) type] == TokenTypeEOF )
|
|
return;
|
|
|
|
NSInteger i = index+1;
|
|
[self sync:i];
|
|
while ( [((CommonToken *)[tokens objectAtIndex:i]) type] != TokenTypeEOF ) {
|
|
i++;
|
|
[self sync:i];
|
|
}
|
|
}
|
|
|
|
@end
|