Spaces:
Sleeping
Sleeping
File size: 6,791 Bytes
3d2101a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | "use strict";
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
// See LICENSE in the project root for license information.
Object.defineProperty(exports, "__esModule", { value: true });
exports.TokenReader = void 0;
const Token_1 = require("./Token");
const TokenSequence_1 = require("./TokenSequence");
/**
* Manages a stream of tokens that are read by the parser.
*
* @remarks
* Use TokenReader.readToken() to read a token and advance the stream pointer.
* Use TokenReader.peekToken() to preview the next token.
* Use TokenReader.createMarker() and backtrackToMarker() to rewind to an earlier point.
* Whenever readToken() is called, the token is added to an accumulated TokenSequence
* that can be extracted by calling extractAccumulatedSequence().
*/
class TokenReader {
constructor(parserContext, embeddedTokenSequence) {
this._parserContext = parserContext;
this.tokens = parserContext.tokens;
if (embeddedTokenSequence) {
if (embeddedTokenSequence.parserContext !== this._parserContext) {
throw new Error('The embeddedTokenSequence must use the same parser context');
}
this._readerStartIndex = embeddedTokenSequence.startIndex;
this._readerEndIndex = embeddedTokenSequence.endIndex;
}
else {
this._readerStartIndex = 0;
this._readerEndIndex = this.tokens.length;
}
this._currentIndex = this._readerStartIndex;
this._accumulatedStartIndex = this._readerStartIndex;
}
/**
* Extracts and returns the TokenSequence that was accumulated so far by calls to readToken().
* The next call to readToken() will start a new accumulated sequence.
*/
extractAccumulatedSequence() {
if (this._accumulatedStartIndex === this._currentIndex) {
// If this happens, it indicates a parser bug:
throw new Error('Parser assertion failed: The queue should not be empty when' +
' extractAccumulatedSequence() is called');
}
const sequence = new TokenSequence_1.TokenSequence({
parserContext: this._parserContext,
startIndex: this._accumulatedStartIndex,
endIndex: this._currentIndex
});
this._accumulatedStartIndex = this._currentIndex;
return sequence;
}
/**
* Returns true if the accumulated sequence has any tokens yet. This will be false
* when the TokenReader starts, and it will be false immediately after a call
* to extractAccumulatedSequence(). Otherwise, it will become true whenever readToken()
* is called.
*/
isAccumulatedSequenceEmpty() {
return this._accumulatedStartIndex === this._currentIndex;
}
/**
* Like extractAccumulatedSequence(), but returns undefined if nothing has been
* accumulated yet.
*/
tryExtractAccumulatedSequence() {
if (this.isAccumulatedSequenceEmpty()) {
return undefined;
}
return this.extractAccumulatedSequence();
}
/**
* Asserts that isAccumulatedSequenceEmpty() should return false. If not, an exception
* is throw indicating a parser bug.
*/
assertAccumulatedSequenceIsEmpty() {
if (!this.isAccumulatedSequenceEmpty()) {
// If this happens, it indicates a parser bug:
const sequence = new TokenSequence_1.TokenSequence({
parserContext: this._parserContext,
startIndex: this._accumulatedStartIndex,
endIndex: this._currentIndex
});
const tokenStrings = sequence.tokens.map((x) => x.toString());
throw new Error('Parser assertion failed: The queue should be empty, but it contains:\n' +
JSON.stringify(tokenStrings));
}
}
/**
* Returns the next token that would be returned by _readToken(), without
* consuming anything.
*/
peekToken() {
return this.tokens[this._currentIndex];
}
/**
* Returns the TokenKind for the next token that would be returned by _readToken(), without
* consuming anything.
*/
peekTokenKind() {
if (this._currentIndex >= this._readerEndIndex) {
return Token_1.TokenKind.EndOfInput;
}
return this.tokens[this._currentIndex].kind;
}
/**
* Like peekTokenKind(), but looks ahead two tokens.
*/
peekTokenAfterKind() {
if (this._currentIndex + 1 >= this._readerEndIndex) {
return Token_1.TokenKind.EndOfInput;
}
return this.tokens[this._currentIndex + 1].kind;
}
/**
* Like peekTokenKind(), but looks ahead three tokens.
*/
peekTokenAfterAfterKind() {
if (this._currentIndex + 2 >= this._readerEndIndex) {
return Token_1.TokenKind.EndOfInput;
}
return this.tokens[this._currentIndex + 2].kind;
}
/**
* Extract the next token from the input stream and return it.
* The token will also be appended to the accumulated sequence, which can
* later be accessed via extractAccumulatedSequence().
*/
readToken() {
if (this._currentIndex >= this._readerEndIndex) {
// If this happens, it's a parser bug
throw new Error('Cannot read past end of stream');
}
const token = this.tokens[this._currentIndex];
if (token.kind === Token_1.TokenKind.EndOfInput) {
// We don't allow reading the EndOfInput token, because we want _peekToken()
// to be always guaranteed to return a valid result.
// If this happens, it's a parser bug
throw new Error('The EndOfInput token cannot be read');
}
this._currentIndex++;
return token;
}
/**
* Returns the kind of the token immediately before the current token.
*/
peekPreviousTokenKind() {
if (this._currentIndex === 0) {
return Token_1.TokenKind.EndOfInput;
}
return this.tokens[this._currentIndex - 1].kind;
}
/**
* Remembers the current position in the stream.
*/
createMarker() {
return this._currentIndex;
}
/**
* Rewinds the stream pointer to a previous position in the stream.
*/
backtrackToMarker(marker) {
if (marker > this._currentIndex) {
// If this happens, it's a parser bug
throw new Error('The marker has expired');
}
this._currentIndex = marker;
if (marker < this._accumulatedStartIndex) {
this._accumulatedStartIndex = marker;
}
}
}
exports.TokenReader = TokenReader;
//# sourceMappingURL=TokenReader.js.map |