Spaces:
Sleeping
Sleeping
File size: 6,582 Bytes
6abff73 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
// See LICENSE in the project root for license information.
import { TokenKind } from './Token';
import { TokenSequence } from './TokenSequence';
/**
* Manages a stream of tokens that are read by the parser.
*
* @remarks
* Use TokenReader.readToken() to read a token and advance the stream pointer.
* Use TokenReader.peekToken() to preview the next token.
* Use TokenReader.createMarker() and backtrackToMarker() to rewind to an earlier point.
* Whenever readToken() is called, the token is added to an accumulated TokenSequence
* that can be extracted by calling extractAccumulatedSequence().
*/
export class TokenReader {
constructor(parserContext, embeddedTokenSequence) {
this._parserContext = parserContext;
this.tokens = parserContext.tokens;
if (embeddedTokenSequence) {
if (embeddedTokenSequence.parserContext !== this._parserContext) {
throw new Error('The embeddedTokenSequence must use the same parser context');
}
this._readerStartIndex = embeddedTokenSequence.startIndex;
this._readerEndIndex = embeddedTokenSequence.endIndex;
}
else {
this._readerStartIndex = 0;
this._readerEndIndex = this.tokens.length;
}
this._currentIndex = this._readerStartIndex;
this._accumulatedStartIndex = this._readerStartIndex;
}
/**
* Extracts and returns the TokenSequence that was accumulated so far by calls to readToken().
* The next call to readToken() will start a new accumulated sequence.
*/
extractAccumulatedSequence() {
if (this._accumulatedStartIndex === this._currentIndex) {
// If this happens, it indicates a parser bug:
throw new Error('Parser assertion failed: The queue should not be empty when' +
' extractAccumulatedSequence() is called');
}
const sequence = new TokenSequence({
parserContext: this._parserContext,
startIndex: this._accumulatedStartIndex,
endIndex: this._currentIndex
});
this._accumulatedStartIndex = this._currentIndex;
return sequence;
}
/**
* Returns true if the accumulated sequence has any tokens yet. This will be false
* when the TokenReader starts, and it will be false immediately after a call
* to extractAccumulatedSequence(). Otherwise, it will become true whenever readToken()
* is called.
*/
isAccumulatedSequenceEmpty() {
return this._accumulatedStartIndex === this._currentIndex;
}
/**
* Like extractAccumulatedSequence(), but returns undefined if nothing has been
* accumulated yet.
*/
tryExtractAccumulatedSequence() {
if (this.isAccumulatedSequenceEmpty()) {
return undefined;
}
return this.extractAccumulatedSequence();
}
/**
* Asserts that isAccumulatedSequenceEmpty() should return false. If not, an exception
* is throw indicating a parser bug.
*/
assertAccumulatedSequenceIsEmpty() {
if (!this.isAccumulatedSequenceEmpty()) {
// If this happens, it indicates a parser bug:
const sequence = new TokenSequence({
parserContext: this._parserContext,
startIndex: this._accumulatedStartIndex,
endIndex: this._currentIndex
});
const tokenStrings = sequence.tokens.map((x) => x.toString());
throw new Error('Parser assertion failed: The queue should be empty, but it contains:\n' +
JSON.stringify(tokenStrings));
}
}
/**
* Returns the next token that would be returned by _readToken(), without
* consuming anything.
*/
peekToken() {
return this.tokens[this._currentIndex];
}
/**
* Returns the TokenKind for the next token that would be returned by _readToken(), without
* consuming anything.
*/
peekTokenKind() {
if (this._currentIndex >= this._readerEndIndex) {
return TokenKind.EndOfInput;
}
return this.tokens[this._currentIndex].kind;
}
/**
* Like peekTokenKind(), but looks ahead two tokens.
*/
peekTokenAfterKind() {
if (this._currentIndex + 1 >= this._readerEndIndex) {
return TokenKind.EndOfInput;
}
return this.tokens[this._currentIndex + 1].kind;
}
/**
* Like peekTokenKind(), but looks ahead three tokens.
*/
peekTokenAfterAfterKind() {
if (this._currentIndex + 2 >= this._readerEndIndex) {
return TokenKind.EndOfInput;
}
return this.tokens[this._currentIndex + 2].kind;
}
/**
* Extract the next token from the input stream and return it.
* The token will also be appended to the accumulated sequence, which can
* later be accessed via extractAccumulatedSequence().
*/
readToken() {
if (this._currentIndex >= this._readerEndIndex) {
// If this happens, it's a parser bug
throw new Error('Cannot read past end of stream');
}
const token = this.tokens[this._currentIndex];
if (token.kind === TokenKind.EndOfInput) {
// We don't allow reading the EndOfInput token, because we want _peekToken()
// to be always guaranteed to return a valid result.
// If this happens, it's a parser bug
throw new Error('The EndOfInput token cannot be read');
}
this._currentIndex++;
return token;
}
/**
* Returns the kind of the token immediately before the current token.
*/
peekPreviousTokenKind() {
if (this._currentIndex === 0) {
return TokenKind.EndOfInput;
}
return this.tokens[this._currentIndex - 1].kind;
}
/**
* Remembers the current position in the stream.
*/
createMarker() {
return this._currentIndex;
}
/**
* Rewinds the stream pointer to a previous position in the stream.
*/
backtrackToMarker(marker) {
if (marker > this._currentIndex) {
// If this happens, it's a parser bug
throw new Error('The marker has expired');
}
this._currentIndex = marker;
if (marker < this._accumulatedStartIndex) {
this._accumulatedStartIndex = marker;
}
}
}
//# sourceMappingURL=TokenReader.js.map |