Buckets:

technician1's picture
download
raw
183 kB
// EnglishLike 2.2 — iterative C++17 compiler and bounded bytecode runtime. // Features: natural-English declarations, grammar-position keyword correction, // functions, classes/structures, // methods, inheritance, objects, includes, compile-time constants, generic // (type-erased) function declarations, ChatIPC-inspired typo assistance. // No recursive C++ algorithms are used in the parser, JSON reader, VM, or // definition expansion. // // Build: g++ -std=c++17 -O3 -DNDEBUG -Wall -Wextra -pedantic -fopenmp EnglishLike.cpp dictionary.cpp -o englishlike // Run: ./englishlike program.el --depth 2 (linked dictionary.cpp is default) #include <algorithm> #include <array> #include <atomic> #include <cctype> #include <cstdint> #include <cstdlib> #include <fstream> #include <iomanip> #include <iostream> #include <limits> #include <memory> #include <random> #include <chrono> #include <cstddef> #include <cmath> #include <cstdio> #include <functional> #include <iterator> #include <optional> #include <sstream> #include <stdexcept> #include <string> #include <string_view> #include <unordered_map> #include <unordered_set> #include <utility> #include <variant> #include <vector> #if defined(_WIN32) // GetFinalPathNameByHandleA is declared by the Windows SDK headers only when // the target API level is Vista (0x0600) or newer. Define the target before // including windows.h; MinGW-w64 GCC 8.1 otherwise hides the declaration. # if !defined(_WIN32_WINNT) || (_WIN32_WINNT < 0x0600) # undef _WIN32_WINNT # define _WIN32_WINNT 0x0600 # endif # if !defined(WINVER) || (WINVER < 0x0600) # undef WINVER # define WINVER 0x0600 # endif # ifndef WIN32_LEAN_AND_MEAN # define WIN32_LEAN_AND_MEAN # endif # ifndef NOMINMAX # define NOMINMAX # endif # include <windows.h> #else # include <limits.h> # include <sys/stat.h> # include <unistd.h> #endif using namespace std; #ifndef ENGLISHLIKE_LIBRARY_ONLY extern unsigned char dictionary_json[]; extern unsigned int dictionary_json_len; #endif namespace englishlike { static constexpr size_t MAX_SOURCE_BYTES = 32u * 1024u * 1024u; static constexpr size_t MAX_DICTIONARY_BYTES = 512u * 1024u * 1024u; static constexpr size_t MAX_DICTIONARY_RECORDS = 1000000u; static constexpr size_t MAX_DEFINITIONS_PER_RECORD = 4096u; static constexpr size_t MAX_TOKENIZED_WORDS = 8192u; static constexpr size_t MAX_DEFINITION_TOKENS_PER_RECORD = 4096u; static constexpr size_t MAX_DEFINITION_CACHE_TOKENS = 2000000u; static constexpr size_t MAX_FINGERPRINT_TOKENS = 8192u; static constexpr size_t MAX_TOKENS = 1u * 1024u * 1024u; static constexpr size_t MAX_JSON_STRING_BYTES = 1u * 1024u * 1024u; static constexpr size_t MAX_IDENTIFIER_BYTES = 4096u; static constexpr size_t MAX_INTEGER_DIGITS = 8192u; static constexpr size_t MAX_ARRAY_ELEMENTS = 262144u; static constexpr size_t MAX_DIAGNOSTICS = 256u; static constexpr size_t MAX_GENERATED_SOURCE_BYTES = 4u * 1024u * 1024u; static constexpr size_t MAX_RUNTIME_STACK = 256u * 1024u; static constexpr size_t MAX_CALL_FRAMES = 256; static constexpr size_t MAX_OBJECTS = 262144u; static constexpr size_t MAX_RUNTIME_OBJECT_FIELDS = 1u * 1024u * 1024u; static constexpr size_t MAX_RUNTIME_HEAP_BYTES = 128u * 1024u * 1024u; static constexpr size_t RUNTIME_SLOT_OVERHEAD = 32u; static constexpr size_t MAX_LOCALS_PER_FUNCTION = 8192u; static constexpr size_t MAX_GLOBALS = 100000u; static constexpr size_t MAX_FUNCTIONS = 16384u; static constexpr size_t MAX_TYPES = 8192u; static constexpr size_t MAX_FIELDS_PER_TYPE = 8192u; static constexpr size_t MAX_PARAMETERS = 1024u; static constexpr uint64_t MAX_OUTPUT_BYTES = 64u * 1024u * 1024u; static constexpr size_t MAX_INPUT_TOKEN_BYTES = 100000u; static constexpr uint64_t DEFAULT_GAS = 10000000ULL; static inline unsigned char u8(char c) noexcept { return static_cast<unsigned char>(c); } static inline char lowerAscii(char c) noexcept { const unsigned char x = u8(c); return (x >= 'A' && x <= 'Z') ? static_cast<char>(x + 32) : c; } static string lowerAscii(string s) { for (char &c : s) c = lowerAscii(c); return s; } static bool isAsciiAlpha(unsigned char c) noexcept { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); } static bool isAsciiDigit(unsigned char c) noexcept { return c >= '0' && c <= '9'; } static bool isIdentifierStart(unsigned char c) noexcept { return isAsciiAlpha(c) || c == '_' || c >= 0x80; } static bool isIdentifierContinue(unsigned char c) noexcept { return isIdentifierStart(c) || isAsciiDigit(c); } static bool isDictionaryKeyChar(unsigned char c) noexcept { return isalnum(c) != 0 || c == 0x27 || c == '-' || c >= 0x80; } static string normalizeDictionaryKey(string_view s) { size_t a=0,b=s.size(); while(a<b && !isDictionaryKeyChar(u8(s[a]))) ++a; while(b>a && !isDictionaryKeyChar(u8(s[b-1]))) --b; string r; r.reserve(b-a); for(size_t i=a;i<b;++i) r.push_back(lowerAscii(s[i])); return r; } static string normalizePosTag(const string &s) { string r; for(char c:s) if(isAsciiAlpha(u8(c))) r.push_back(lowerAscii(c)); return r; } // Per-process keyed SipHash-2-4 reduces hash-flooding risk for untrusted // identifiers and dictionary keys. The key is initialized once at startup. static uint64_t rotl64(uint64_t x,unsigned n) noexcept { return (x<<n)|(x>>(64-n)); } struct KeyedStringHash { static pair<uint64_t,uint64_t> keys() noexcept { static const pair<uint64_t,uint64_t> value=[]() noexcept { uint64_t a=0,b=0; try { random_device rd; a=(static_cast<uint64_t>(rd())<<32)^rd(); b=(static_cast<uint64_t>(rd())<<32)^rd(); } catch (...) { const auto t=static_cast<uint64_t>(chrono::high_resolution_clock::now().time_since_epoch().count()); a=t^0x9e3779b97f4a7c15ULL; b=rotl64(t,29)^0xd1b54a32d192ed03ULL; } if((a|b)==0){a=0x243f6a8885a308d3ULL;b=0x13198a2e03707344ULL;} return make_pair(a,b); }(); return value; } size_t operator()(const string&text) const noexcept { const auto key=keys();uint64_t v0=0x736f6d6570736575ULL^key.first,v1=0x646f72616e646f6dULL^key.second; uint64_t v2=0x6c7967656e657261ULL^key.first,v3=0x7465646279746573ULL^key.second; auto round=[&]() noexcept {v0+=v1;v1=rotl64(v1,13);v1^=v0;v0=rotl64(v0,32);v2+=v3;v3=rotl64(v3,16);v3^=v2;v0+=v3;v3=rotl64(v3,21);v3^=v0;v2+=v1;v1=rotl64(v1,17);v1^=v2;v2=rotl64(v2,32);}; size_t i=0;while(i+8<=text.size()){uint64_t m=0;for(unsigned j=0;j<8;++j)m|=static_cast<uint64_t>(u8(text[i+j]))<<(8*j);v3^=m;round();round();v0^=m;i+=8;} uint64_t tail=static_cast<uint64_t>(text.size())<<56;for(unsigned j=0;i+j<text.size();++j)tail|=static_cast<uint64_t>(u8(text[i+j]))<<(8*j); v3^=tail;round();round();v0^=tail;v2^=0xff;round();round();round();round();return static_cast<size_t>(v0^v1^v2^v3); } }; using StringSet=unordered_set<string,KeyedStringHash>; template<class T> using StringMap=unordered_map<string,T,KeyedStringHash>; // Iterative heapsort. The compiler deliberately avoids recursive sorting // helpers so nested/untrusted inputs cannot consume the C++ call stack. template<class RandomIt, class Compare> static void iterativeHeapSort(RandomIt first, RandomIt last, Compare comp) { const size_t count = static_cast<size_t>(last - first); auto siftDown = [&](size_t root, size_t length) { while (root < length / 2) { size_t child = root * 2 + 1; if (child + 1 < length && comp(*(first + child), *(first + child + 1))) ++child; if (!comp(*(first + root), *(first + child))) break; iter_swap(first + root, first + child); root = child; } }; for (size_t start = count / 2; start > 0; --start) siftDown(start - 1, count); for (size_t end = count; end > 1; --end) { iter_swap(first, first + (end - 1)); siftDown(0, end - 1); } } // ------------------------------ JSON reader ------------------------------ // Reads a byte span, not a C string. The byte count is authoritative; embedded // JSON need not contain a trailing NUL. String escapes, including UTF-16 // surrogate pairs, are decoded to UTF-8. Unknown JSON fields are skipped safely. struct DictionaryEntry { string word; string pos; vector<string> definitions; }; class JsonReader { public: explicit JsonReader(string_view source) : source_(source) {} vector<DictionaryEntry> parseDictionary() { skipWhitespace(); expect('['); vector<DictionaryEntry> entries; skipWhitespace(); if (consume(']')) { requireEnd(); return entries; } for (;;) { skipWhitespace(); if (peek() != '{') fail("dictionary array entries must be objects"); if (entries.size() >= MAX_DICTIONARY_RECORDS) fail("dictionary entry-count limit exceeded"); entries.push_back(parseEntry()); skipWhitespace(); if (consume(']')) break; expect(','); } requireEnd(); return entries; } private: string_view source_; size_t position_ = 0; static constexpr unsigned int MAX_NESTING = 256; [[noreturn]] void fail(const char *message) const { throw runtime_error(string("dictionary JSON at byte ") + to_string(position_) + ": " + message); } char peek() const noexcept { return position_ < source_.size() ? source_[position_] : '\0'; } bool consume(char c) noexcept { if (position_ < source_.size() && source_[position_] == c) { ++position_; return true; } return false; } void expect(char c) { if (!consume(c)) fail("unexpected character"); } void skipWhitespace() noexcept { while (position_ < source_.size() && (source_[position_] == ' ' || source_[position_] == '\t' || source_[position_] == '\n' || source_[position_] == '\r')) { ++position_; } } void requireEnd() { skipWhitespace(); if (position_ != source_.size()) fail("trailing data after root JSON value"); } static void appendUtf8(string &out, uint32_t cp) { if (cp <= 0x7F) { out.push_back(static_cast<char>(cp)); } else if (cp <= 0x7FF) { out.push_back(static_cast<char>(0xC0 | (cp >> 6))); out.push_back(static_cast<char>(0x80 | (cp & 0x3F))); } else if (cp <= 0xFFFF) { out.push_back(static_cast<char>(0xE0 | (cp >> 12))); out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F))); out.push_back(static_cast<char>(0x80 | (cp & 0x3F))); } else { out.push_back(static_cast<char>(0xF0 | (cp >> 18))); out.push_back(static_cast<char>(0x80 | ((cp >> 12) & 0x3F))); out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F))); out.push_back(static_cast<char>(0x80 | (cp & 0x3F))); } } uint32_t readHexQuad() { uint32_t value = 0; for (int i = 0; i < 4; ++i) { if (position_ >= source_.size()) fail("truncated Unicode escape"); const unsigned char c = u8(source_[position_++]); value <<= 4; if (c >= '0' && c <= '9') value |= c - '0'; else if (c >= 'a' && c <= 'f') value |= c - 'a' + 10; else if (c >= 'A' && c <= 'F') value |= c - 'A' + 10; else fail("invalid hexadecimal digit in Unicode escape"); } return value; } string parseString() { expect('"'); string result; while (position_ < source_.size()) { const unsigned char c = u8(source_[position_++]); if (c == '"') return result; if (c < 0x20) fail("unescaped control character in JSON string"); if (c != '\\') { result.push_back(static_cast<char>(c)); if (result.size() > MAX_JSON_STRING_BYTES) fail("JSON string exceeds 1 MiB safety limit"); continue; } if (position_ >= source_.size()) fail("truncated escape sequence"); const char escape = source_[position_++]; switch (escape) { case '"': result.push_back('"'); break; case '\\': result.push_back('\\'); break; case '/': result.push_back('/'); break; case 'b': result.push_back('\b'); break; case 'f': result.push_back('\f'); break; case 'n': result.push_back('\n'); break; case 'r': result.push_back('\r'); break; case 't': result.push_back('\t'); break; case 'u': { uint32_t cp = readHexQuad(); if (cp >= 0xD800 && cp <= 0xDBFF) { if (position_ + 2 > source_.size() || source_[position_] != '\\' || source_[position_ + 1] != 'u') { fail("high surrogate is not followed by a low surrogate"); } position_ += 2; const uint32_t low = readHexQuad(); if (low < 0xDC00 || low > 0xDFFF) fail("invalid low surrogate"); cp = 0x10000 + ((cp - 0xD800) << 10) + (low - 0xDC00); } else if (cp >= 0xDC00 && cp <= 0xDFFF) { fail("unpaired low surrogate"); } appendUtf8(result, cp); break; } default: fail("invalid JSON escape sequence"); } if (result.size() > MAX_JSON_STRING_BYTES) fail("JSON string exceeds 1 MiB safety limit"); } fail("unterminated JSON string"); } DictionaryEntry parseEntry() { DictionaryEntry entry; bool seenWord = false; bool seenPos = false; bool seenDefinitions = false; expect('{'); skipWhitespace(); if (consume('}')) fail("dictionary entry has no word field"); for (;;) { skipWhitespace(); if (peek() != '"') fail("object field name must be a string"); const string key = parseString(); skipWhitespace(); expect(':'); skipWhitespace(); if (key == "word") { if (seenWord) fail("duplicate word field"); seenWord = true; if (peek() != '"') fail("word field must be a string"); entry.word = parseString(); } else if (key == "pos") { if (seenPos) fail("duplicate pos field"); seenPos = true; if (peek() != '"') fail("pos field must be a string"); entry.pos = parseString(); } else if (key == "definitions") { if (seenDefinitions) fail("duplicate definitions field"); seenDefinitions = true; entry.definitions = parseStringArray(); } else { skipValue(0); } skipWhitespace(); if (consume('}')) break; expect(','); } if (!seenWord || entry.word.empty()) fail("dictionary entry requires a non-empty word field"); return entry; } vector<string> parseStringArray() { vector<string> values; expect('['); skipWhitespace(); if (consume(']')) return values; for (;;) { skipWhitespace(); if (peek() != '"') fail("definitions must be an array of strings"); if (values.size() >= MAX_DEFINITIONS_PER_RECORD) fail("definition-count limit exceeded for dictionary record"); values.push_back(parseString()); skipWhitespace(); if (consume(']')) break; expect(','); } return values; } void skipValue(unsigned int /*depth*/) { struct Frame { char kind; unsigned char state; }; vector<Frame> stack; auto startValue = [&]() { skipWhitespace(); const char c = peek(); if (c == '{' || c == '[') { ++position_; if (stack.size() >= MAX_NESTING) fail("maximum JSON nesting depth exceeded"); stack.push_back({c, 0}); } else if (c == '"') { (void)parseString(); } else if (c == 't') consumeLiteral("true"); else if (c == 'f') consumeLiteral("false"); else if (c == 'n') consumeLiteral("null"); else skipNumber(); }; startValue(); while (!stack.empty()) { skipWhitespace(); Frame &frame = stack.back(); if (frame.kind == '[') { if (frame.state == 0) { if (consume(']')) { stack.pop_back(); continue; } frame.state = 1; startValue(); } else { if (consume(']')) { stack.pop_back(); continue; } expect(','); frame.state = 0; } } else { if (frame.state == 0) { if (consume('}')) { stack.pop_back(); continue; } if (peek() != '"') fail("object field name must be a string"); (void)parseString(); frame.state = 1; } else if (frame.state == 1) { expect(':'); frame.state = 2; } else if (frame.state == 2) { frame.state = 3; startValue(); } else { if (consume('}')) { stack.pop_back(); continue; } expect(','); frame.state = 0; } } } } void consumeLiteral(const char *literal) { for (const char *p = literal; *p; ++p) { if (position_ >= source_.size() || source_[position_++] != *p) fail("invalid JSON literal"); } } void skipNumber() { const size_t start = position_; consume('-'); if (consume('0')) { // A leading zero cannot be followed by another integer digit. if (position_ < source_.size() && isAsciiDigit(u8(source_[position_]))) fail("leading zero in JSON number"); } else { const size_t digits = position_; while (position_ < source_.size() && isAsciiDigit(u8(source_[position_]))) ++position_; if (digits == position_) fail("invalid JSON value"); } if (consume('.')) { const size_t fraction = position_; while (position_ < source_.size() && isAsciiDigit(u8(source_[position_]))) ++position_; if (fraction == position_) fail("invalid JSON fraction"); } if (consume('e') || consume('E')) { if (!consume('+')) consume('-'); const size_t exponent = position_; while (position_ < source_.size() && isAsciiDigit(u8(source_[position_]))) ++position_; if (exponent == position_) fail("invalid JSON exponent"); } if (start == position_) fail("invalid JSON value"); } }; static bool dictionaryContains(const StringSet &words, const string &surface) { const string key = normalizeDictionaryKey(surface); return !key.empty() && words.find(key) != words.end(); } static vector<string> digitToWords(const string &input) { static const array<const char *, 20> ones = {{ "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen" }}; static const array<const char *, 10> tens = {{ "zero", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety" }}; static const array<const char *, 24> scales = {{ "", "thousand", "million", "billion", "trillion", "quadrillion", "quintillion", "sextillion", "septillion", "octillion", "nonillion", "decillion", "undecillion", "duodecillion", "tredecillion", "quattuordecillion", "quindecillion", "sexdecillion", "septendecillion", "octodecillion", "novemdecillion", "vigintillion", "googol", "" }}; string integerPart = input; string fractionalPart; const size_t dot = input.find('.'); if (dot != string::npos) { integerPart = input.substr(0, dot); fractionalPart = input.substr(dot + 1); } string digits; for (char c : integerPart) if (isAsciiDigit(u8(c))) digits.push_back(c); vector<string> output; if (digits.size() > 4096 || fractionalPart.size() > 4096) { output.emplace_back("number"); return output; } if (digits.empty()) { if (fractionalPart.empty()) return output; output.emplace_back("zero"); } else { const size_t first = digits.find_first_not_of('0'); if (first == string::npos) { output.emplace_back("zero"); } else { digits.erase(0, first); if (digits.size() > 69) { for (char c : digits) output.emplace_back(ones[static_cast<size_t>(c - '0')]); } else { vector<vector<string>> groups; size_t scale = 0; while (!digits.empty()) { const size_t begin = digits.size() > 3 ? digits.size() - 3 : 0; const unsigned int chunk = static_cast<unsigned int>(stoul(digits.substr(begin))); digits.erase(begin); vector<string> chunkWords; if (chunk != 0) { const unsigned int h = chunk / 100; const unsigned int rem = chunk % 100; if (h) { chunkWords.emplace_back(ones[h]); chunkWords.emplace_back("hundred"); } if (rem) { if (rem < 20) chunkWords.emplace_back(ones[rem]); else { chunkWords.emplace_back(tens[rem / 10]); if (rem % 10) chunkWords.emplace_back(ones[rem % 10]); } } if (scale > 0 && scale < scales.size()) chunkWords.emplace_back(scales[scale]); } groups.push_back(move(chunkWords)); ++scale; } for (auto group = groups.rbegin(); group != groups.rend(); ++group) output.insert(output.end(), group->begin(), group->end()); } } } bool hasFractionDigit = false; for (char c : fractionalPart) if (isAsciiDigit(u8(c))) hasFractionDigit = true; if (hasFractionDigit) { output.emplace_back("point"); for (char c : fractionalPart) if (isAsciiDigit(u8(c))) output.emplace_back(ones[static_cast<size_t>(c - '0')]); } return output; } static bool isAllFormattedNumber(const string &s) { bool hasDigit = false; for (char c : s) { const unsigned char u = u8(c); if (isAsciiDigit(u)) hasDigit = true; else if (c != ',' && c != '.') return false; } return !s.empty() && hasDigit; } static string lowerCopy(const string &s) { string result; result.reserve(s.size()); for (char c : s) result.push_back(lowerAscii(c)); return result; } static string validBaseForm(const string &word, const StringSet &dictionary) { if (word.empty()) return {}; if (dictionaryContains(dictionary, word)) return lowerCopy(word); auto found = [&](const string &candidate) { return dictionaryContains(dictionary, candidate); }; const size_t n = word.size(); if (n > 3 && word.compare(n - 3, 3, "ing") == 0) { const string a = word.substr(0, n - 3); if (found(a)) return a; if (found(a + "e")) return a + "e"; if (a.size() >= 2 && a.back() == a[a.size() - 2]) { const string b = a.substr(0, a.size() - 1); if (found(b)) return b; } } if (n > 2 && word.compare(n - 2, 2, "ed") == 0) { const string a = word.substr(0, n - 1); const string b = word.substr(0, n - 2); if (found(a)) return a; if (found(b)) return b; if (b.size() >= 2 && b.back() == b[b.size() - 2]) { const string c = b.substr(0, b.size() - 1); if (found(c)) return c; } } if (n > 3 && word.compare(n - 3, 3, "est") == 0) { const string a = word.substr(0, n - 2); const string b = word.substr(0, n - 3); if (found(a)) return a; if (found(b)) return b; if (n > 4 && word.compare(n - 4, 4, "iest") == 0) { const string y = word.substr(0, n - 4) + "y"; if (found(y)) return y; } if (b.size() >= 2 && b.back() == b[b.size() - 2]) { const string c = b.substr(0, b.size() - 1); if (found(c)) return c; } } if (n > 2 && word.compare(n - 2, 2, "er") == 0) { const string a = word.substr(0, n - 1); const string b = word.substr(0, n - 2); if (found(a)) return a; if (found(b)) return b; if (n > 3 && word.compare(n - 3, 3, "ier") == 0) { const string y = word.substr(0, n - 3) + "y"; if (found(y)) return y; } if (b.size() >= 2 && b.back() == b[b.size() - 2]) { const string c = b.substr(0, b.size() - 1); if (found(c)) return c; } } if (n > 2 && word.compare(n - 2, 2, "ly") == 0) { const string a = word.substr(0, n - 2); if (found(a)) return a; if (n > 3 && word.compare(n - 3, 3, "ily") == 0) { const string b = word.substr(0, n - 3) + "y"; if (found(b)) return b; } if (n > 4 && word.compare(n - 4, 4, "ally") == 0) { const string b = word.substr(0, n - 4); if (found(b)) return b; if (found(b + "al")) return b + "al"; } if (word.back() == 'y' && n > 2 && word[n - 2] == 'l') { const string b = word.substr(0, n - 1) + "e"; if (found(b)) return b; } } if (n > 4 && word.compare(n - 4, 4, "ness") == 0) { const string a = word.substr(0, n - 4); if (found(a)) return a; if (a.size() > 1 && a.back() == 'i') { const string b = a.substr(0, a.size() - 1) + "y"; if (found(b)) return b; } } if (n > 4 && word.compare(n - 4, 4, "ment") == 0) { const string a = word.substr(0, n - 4); if (found(a)) return a; } if (n > 3 && word.compare(n - 3, 3, "ies") == 0) { const string a = word.substr(0, n - 3) + "y"; if (found(a)) return a; } if (n > 3 && word.compare(n - 3, 3, "ves") == 0) { const string stem = word.substr(0, n - 3); if (found(stem + "f")) return stem + "f"; if (found(stem + "fe")) return stem + "fe"; } if (n > 2 && word.compare(n - 2, 2, "es") == 0) { const string a = word.substr(0, n - 2); if (found(a)) return a; } if (n > 1 && word.back() == 's' && word[n - 2] != 's') { const string a = word.substr(0, n - 1); if (found(a)) return a; } return {}; } static vector<string> tokenizeOthers( const string &source, const StringSet &dictionary) { if (dictionaryContains(dictionary, source)) return {lowerCopy(source)}; if (isAllFormattedNumber(source)) return digitToWords(source); vector<string> output; output.reserve(min<size_t>(source.size() / 4 + 1, MAX_TOKENIZED_WORDS)); string current; auto append = [&](string token) { if (output.size() < MAX_TOKENIZED_WORDS) output.push_back(move(token)); }; auto flush = [&]() { if (current.empty()) return; if (isAllFormattedNumber(current)) { vector<string> words = digitToWords(current); for (string &word : words) append(move(word)); } else { const string base = validBaseForm(current, dictionary); if (!base.empty()) append(base); } current.clear(); }; for (size_t i = 0; i < source.size() && output.size() < MAX_TOKENIZED_WORDS; ++i) { const unsigned char c = u8(source[i]); const bool punctuation = ispunct(c) != 0; if (punctuation) { if ((source[i] == ',' || source[i] == '.') && !current.empty() && (isAsciiDigit(u8(current.back())) || current.back() == ',' || current.back() == '.') && i + 1 < source.size() && isAsciiDigit(u8(source[i + 1]))) { current.push_back(source[i]); continue; } flush(); string symbol(1, source[i]); if (i + 1 < source.size() && ispunct(u8(source[i + 1])) != 0) { const string pair = symbol + source[i + 1]; if (dictionaryContains(dictionary, pair)) { append(pair); ++i; continue; } } if (dictionaryContains(dictionary, symbol)) append(symbol); continue; } if (isspace(c) != 0) { flush(); continue; } bool camelBoundary = false; if (isupper(c) != 0 && !current.empty()) { const unsigned char previous = u8(source[i - 1]); const bool previousLowerOrDigit = islower(previous) != 0 || isAsciiDigit(previous); const bool previousUpper = isupper(previous) != 0; const bool nextLower = i + 1 < source.size() && islower(u8(source[i + 1])) != 0; camelBoundary = previousLowerOrDigit || (previousUpper && nextLower); } bool digitBoundary = false; if (!current.empty()) { const unsigned char previous = u8(source[i - 1]); const bool previousDigit = isAsciiDigit(previous) || previous == ',' || previous == '.'; const bool currentDigit = isAsciiDigit(c); digitBoundary = previousDigit != currentDigit; } if (camelBoundary || digitBoundary) flush(); if (isalnum(c) != 0 || c >= 0x80) current.push_back(lowerAscii(source[i])); else flush(); } flush(); return output; } class DefinitionModel { public: explicit DefinitionModel(vector<DictionaryEntry> entries) : entries_(move(entries)) { dictionaryWords_.reserve(entries_.size() * 2 + 1); for (const DictionaryEntry &entry : entries_) { size_t begin = 0; while (begin <= entry.word.size()) { const size_t end = entry.word.find_first_of(";,", begin); const string_view part(entry.word.data() + begin, (end == string::npos ? entry.word.size() : end) - begin); const string key = normalizeDictionaryKey(part); // ChatIPC's check_dictionary() consults global_pos_cache, so // a word is considered present only when it has a POS tag. if (!key.empty() && !normalizePosTag(entry.pos).empty()) dictionaryWords_.insert(key); if (end == string::npos) break; begin = end + 1; } } buildDefinitionCache(); } const StringSet &dictionaryWords() const noexcept { return dictionaryWords_; } vector<string> tokenize(const string &s) const { return tokenizeOthers(s, dictionaryWords_); } StringSet fingerprint(const string &surface, int depth) const { StringSet result; const string root = normalizeDictionaryKey(surface); if (!root.empty()) result.insert(root); addDepthExpansion(root, depth, result); // Mirror ChatIPC's candidate/context sub-token pass: add each result of // tokenize_others and that sub-token's own depth-limited definitions. const vector<string> parts = tokenizeOthers(surface, dictionaryWords_); for (const string &part : parts) { const string key = normalizeDictionaryKey(part); if (key.empty()) continue; if (result.size() >= MAX_FINGERPRINT_TOKENS) break; result.insert(key); addDepthExpansion(key, depth, result); } return result; } double jaccard(const StringSet &a, const StringSet &b) const { if (a.empty() && b.empty()) return 1.0; const auto *small = &a; const auto *large = &b; if (small->size() > large->size()) swap(small, large); size_t intersection = 0; for (const string &token : *small) if (large->find(token) != large->end()) ++intersection; const size_t unionSize = a.size() + b.size() - intersection; return unionSize ? static_cast<double>(intersection) / static_cast<double>(unionSize) : 0.0; } private: vector<DictionaryEntry> entries_; StringSet dictionaryWords_; StringMap<vector<string>> definitionTokens_; void buildDefinitionCache() { const ptrdiff_t count = static_cast<ptrdiff_t>(entries_.size()); atomic<size_t> cacheBudget{0}; #ifdef _OPENMP #pragma omp parallel if(count > 128) #endif { StringMap<vector<string>> local; #ifdef _OPENMP #pragma omp for schedule(guided) #endif for (ptrdiff_t i = 0; i < count; ++i) { if (cacheBudget.load(memory_order_relaxed) >= MAX_DEFINITION_CACHE_TOKENS) continue; const DictionaryEntry &entry = entries_[static_cast<size_t>(i)]; vector<string> allDefinitionTokens; allDefinitionTokens.reserve(128); for (const string &definition : entry.definitions) { if (allDefinitionTokens.size() >= MAX_DEFINITION_TOKENS_PER_RECORD) break; vector<string> tokens = tokenizeOthers(definition, dictionaryWords_); const size_t room = MAX_DEFINITION_TOKENS_PER_RECORD - allDefinitionTokens.size(); const size_t take = min(room, tokens.size()); allDefinitionTokens.insert(allDefinitionTokens.end(), tokens.begin(), tokens.begin() + static_cast<ptrdiff_t>(take)); } size_t begin = 0; while (begin <= entry.word.size() && cacheBudget.load(memory_order_relaxed) < MAX_DEFINITION_CACHE_TOKENS) { const size_t end = entry.word.find_first_of(";,", begin); const string_view part(entry.word.data() + begin, (end == string::npos ? entry.word.size() : end) - begin); const string key = normalizeDictionaryKey(part); if (!key.empty()) { auto &destination = local[key]; for (const string &token : allDefinitionTokens) { size_t used = cacheBudget.load(memory_order_relaxed); bool claimed = false; while (used < MAX_DEFINITION_CACHE_TOKENS) { if (cacheBudget.compare_exchange_weak(used, used + 1, memory_order_relaxed)) { claimed = true; break; } } if (!claimed) break; destination.push_back(token); } } if (end == string::npos) break; begin = end + 1; } } #ifdef _OPENMP #pragma omp critical(englishlike_merge_definitions) #endif { for (auto &item : local) { auto &destination = definitionTokens_[item.first]; destination.insert(destination.end(), make_move_iterator(item.second.begin()), make_move_iterator(item.second.end())); } } } vector<string> keys; keys.reserve(definitionTokens_.size()); for (const auto &item : definitionTokens_) keys.push_back(item.first); #ifdef _OPENMP #pragma omp parallel for schedule(dynamic) if(keys.size() > 128) #endif for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(keys.size()); ++i) { auto &tokens = definitionTokens_.find(keys[static_cast<size_t>(i)])->second; iterativeHeapSort(tokens.begin(), tokens.end(), less<string>{}); tokens.erase(unique(tokens.begin(), tokens.end()), tokens.end()); } } void addDepthExpansion(const string &start, int depth, StringSet &result) const { if (start.empty() || depth <= 0 || result.size() >= MAX_FINGERPRINT_TOKENS) return; auto first = definitionTokens_.find(start); if (first == definitionTokens_.end()) return; vector<string> frontier; frontier.reserve(first->second.size()); for (const string &token : first->second) { if (result.size() < MAX_FINGERPRINT_TOKENS && result.insert(token).second) frontier.push_back(token); } for (int level = 1; level < depth && !frontier.empty(); ++level) { vector<string> next; for (const string &word : frontier) { const string key = normalizeDictionaryKey(word); const auto found = definitionTokens_.find(key); if (found == definitionTokens_.end()) continue; for (const string &token : found->second) { if (result.size() < MAX_FINGERPRINT_TOKENS && result.insert(token).second) next.push_back(token); } } frontier.swap(next); } } }; class BigNat { public: BigNat() : limbs_(1, 0) {} explicit BigNat(uint64_t value) { do { limbs_.push_back(static_cast<uint32_t>(value % BASE)); value /= BASE; } while (value); } explicit BigNat(string_view decimal) { if (decimal.empty()) throw runtime_error("empty integer literal"); if (decimal.size() > MAX_INTEGER_DIGITS) throw runtime_error("integer literal exceeds the 8192-digit safety limit"); for (char c : decimal) if (!isAsciiDigit(u8(c))) throw runtime_error("invalid natural-number literal"); for (size_t end = decimal.size(); end > 0;) { const size_t begin = end > 9 ? end - 9 : 0; uint32_t limb = 0; for (size_t i = begin; i < end; ++i) limb = limb * 10u + static_cast<uint32_t>(decimal[i] - '0'); limbs_.push_back(limb); end = begin; } trim(); } bool isZero() const noexcept { return limbs_.size() == 1 && limbs_[0] == 0; } size_t limbCount() const noexcept { return limbs_.size(); } string toString() const { string out = to_string(limbs_.back()); for (size_t i = limbs_.size() - 1; i-- > 0;) { const string part = to_string(limbs_[i]); out.append(9 - part.size(), '0'); out += part; } return out; } friend int compare(const BigNat &a, const BigNat &b) noexcept { if (a.limbs_.size() != b.limbs_.size()) return a.limbs_.size() < b.limbs_.size() ? -1 : 1; for (size_t i = a.limbs_.size(); i-- > 0;) { if (a.limbs_[i] != b.limbs_[i]) return a.limbs_[i] < b.limbs_[i] ? -1 : 1; } return 0; } friend BigNat add(const BigNat &a, const BigNat &b) { BigNat result; result.limbs_.assign(max(a.limbs_.size(), b.limbs_.size()) + 1, 0); uint64_t carry = 0; for (size_t i = 0; i + 1 < result.limbs_.size(); ++i) { const uint64_t av = i < a.limbs_.size() ? a.limbs_[i] : 0; const uint64_t bv = i < b.limbs_.size() ? b.limbs_[i] : 0; const uint64_t sum = av + bv + carry; result.limbs_[i] = static_cast<uint32_t>(sum % BASE); carry = sum / BASE; } result.limbs_.back() = static_cast<uint32_t>(carry); result.trim(); result.enforceDigitLimit(); return result; } friend BigNat subtract(const BigNat &a, const BigNat &b) { if (compare(a, b) < 0) throw runtime_error("natural-number subtraction underflow"); BigNat result; result.limbs_.assign(a.limbs_.size(), 0); int64_t borrow = 0; for (size_t i = 0; i < a.limbs_.size(); ++i) { int64_t value = static_cast<int64_t>(a.limbs_[i]) - borrow - (i < b.limbs_.size() ? static_cast<int64_t>(b.limbs_[i]) : 0); if (value < 0) { value += static_cast<int64_t>(BASE); borrow = 1; } else borrow = 0; result.limbs_[i] = static_cast<uint32_t>(value); } result.trim(); return result; } friend BigNat multiply(const BigNat &a, const BigNat &b) { BigNat result; result.limbs_.assign(a.limbs_.size() + b.limbs_.size(), 0); for (size_t i = 0; i < a.limbs_.size(); ++i) { uint64_t carry = 0; for (size_t j = 0; j < b.limbs_.size() || carry; ++j) { const uint64_t current = result.limbs_[i + j] + carry + (j < b.limbs_.size() ? static_cast<uint64_t>(a.limbs_[i]) * b.limbs_[j] : 0ULL); result.limbs_[i + j] = static_cast<uint32_t>(current % BASE); carry = current / BASE; } } result.trim(); result.enforceDigitLimit(); return result; } private: static constexpr uint64_t BASE = 1000000000ULL; vector<uint32_t> limbs_; void enforceDigitLimit() const { const size_t digits = (limbs_.size() - 1) * 9 + to_string(limbs_.back()).size(); if (digits > MAX_INTEGER_DIGITS) throw runtime_error("natural-number result exceeds the 8192-digit safety limit"); } void trim() { while (limbs_.size() > 1 && limbs_.back() == 0) limbs_.pop_back(); } }; static double parseFiniteFloat(const string &text) { size_t consumed = 0; double value = 0.0; try { value = stod(text, &consumed); } catch (...) { throw runtime_error("invalid floating-point literal"); } if (consumed != text.size() || !isfinite(value) || fabs(value) > 1.0e300) throw runtime_error("floating-point literal is outside the supported finite range"); return value; } // -------------------------------- Lexer ----------------------------------- enum class TokenKind : uint8_t { Word, Number, String, Symbol, End }; struct Token { TokenKind kind=TokenKind::End; string text; size_t line=1, column=1; }; class Lexer { public: explicit Lexer(string_view source) : source_(source) {} vector<Token> scan() { vector<Token> out; out.reserve(min<size_t>(source_.size()/2+1, MAX_TOKENS)); while (pos_ < source_.size()) { const unsigned char c=u8(source_[pos_]); if (isspace(c)) { advance(); continue; } if (source_[pos_]=='#' || (source_[pos_]=='/' && pos_+1<source_.size() && source_[pos_+1]=='/')) { skipLine(); continue; } if (source_[pos_]=='/' && pos_+1<source_.size() && source_[pos_+1]=='*') { skipBlockComment(); continue; } const size_t line=line_, col=col_; if (isIdentifierStart(c)) { string word; while(pos_<source_.size() && isIdentifierContinue(u8(source_[pos_]))) { if (word.size() >= MAX_IDENTIFIER_BYTES) fail("identifier exceeds 4096-byte safety limit"); word.push_back(source_[pos_]); advance(); } add(out,{TokenKind::Word,move(word),line,col}); } else if (isAsciiDigit(c)) { string number; bool decimalPoint = false; while (pos_ < source_.size()) { const char current = source_[pos_]; if (isAsciiDigit(u8(current)) || current == '_') { if (current != '_') { if (number.size() >= MAX_INTEGER_DIGITS) fail("numeric literal exceeds 8192-byte safety limit"); number.push_back(current); } advance(); } else if (current == '.' && !decimalPoint && pos_ + 1 < source_.size() && isAsciiDigit(u8(source_[pos_ + 1]))) { decimalPoint = true; number.push_back('.'); advance(); } else break; } add(out,{TokenKind::Number,move(number),line,col}); } else if (source_[pos_]=='"') { add(out,{TokenKind::String,scanString(),line,col}); } else { string symbol(1,source_[pos_]); if (pos_+1<source_.size()) { const string two(source_.substr(pos_,2)); if(two=="=="||two=="!="||two=="<="||two==">="||two=="&&"||two=="||") symbol=two; } if(string_view(";(),+-*/%=!<>(){}[]:").find(source_[pos_])==string_view::npos) fail("unsupported character"); for(size_t i=0;i<symbol.size();++i) advance(); add(out,{TokenKind::Symbol,move(symbol),line,col}); } } out.push_back({TokenKind::End,"",line_,col_}); return out; } private: string_view source_; size_t pos_=0,line_=1,col_=1; void add(vector<Token>&v,Token t){ if(v.size()>=MAX_TOKENS) fail("token count limit exceeded"); v.push_back(move(t)); } [[noreturn]] void fail(const char*m)const{throw runtime_error("lexical error at "+to_string(line_)+":"+to_string(col_)+": "+m);} void advance(){ if(source_[pos_++]=='\n'){++line_;col_=1;}else ++col_; } void skipLine(){while(pos_<source_.size()&&source_[pos_]!='\n')advance();} void skipBlockComment(){advance();advance();while(pos_+1<source_.size()&&!(source_[pos_]=='*'&&source_[pos_+1]=='/'))advance();if(pos_+1>=source_.size())fail("unterminated block comment");advance();advance();} string scanString(){ advance(); string out; out.reserve(32); while(pos_<source_.size()){ char c=source_[pos_];advance(); if(c=='"')return out; if(c=='\n'||c=='\r')fail("newline in string literal"); if(c=='\\'){ if(pos_>=source_.size())fail("truncated string escape"); const char e=source_[pos_];advance(); switch(e){case 'n':out.push_back('\n');break;case 'r':out.push_back('\r');break;case 't':out.push_back('\t');break;case '\\':out.push_back('\\');break;case '"':out.push_back('"');break;default:fail("unsupported string escape");} } else out.push_back(c); if(out.size()>1024u*1024u)fail("string literal exceeds 1 MiB limit"); } fail("unterminated string literal"); } }; // ------------------------------ Values/IR -------------------------------- struct ObjectRef { uint32_t id=0; }; using Value = variant<BigNat,double,bool,string,ObjectRef>; enum class Op : uint8_t { PushNumber, PushString, PushBoolean, LoadLocal, StoreLocal, LoadGlobal, StoreGlobal, LoadThis, NewObject, NewArray, NewMap, GetField, SetField, ArrayGet, ArraySet, ArrayLength, ArrayAppend, Add, Subtract, Multiply, Divide, Remainder, Equal, NotEqual, Less, LessEqual, Greater, GreaterEqual, BooleanAnd, BooleanOr, BooleanNot, Jump, JumpIfFalse, Call, CallMethod, Return, Print, Read, Pop, PushHandler, PopHandler, Throw }; struct Instruction { Op op=Op::Pop; size_t a=0,b=0; }; enum class AccessLevel : uint8_t { Public, Protected, Private }; enum class StaticType : uint8_t { Unknown, Natural, Floating, Boolean, String, Object, Array, Map }; struct FunctionInfo { string name; vector<string> parameters, localNames; StringMap<size_t> localIds; StringMap<size_t> localObjectTypes; vector<StaticType> localTypes; vector<Instruction> code; size_t ownerType=numeric_limits<size_t>::max(); bool generic=false; vector<string> genericParameters; AccessLevel access=AccessLevel::Public; StaticType returnType=StaticType::Unknown; vector<StaticType> parameterTypes; vector<string> parameterTypeNames; }; struct FieldInfo { string name; Value initial=BigNat(0); AccessLevel access=AccessLevel::Public; StaticType type=StaticType::Unknown; size_t declaringType=numeric_limits<size_t>::max(); }; struct TypeInfo { string name; size_t base=numeric_limits<size_t>::max(); vector<FieldInfo> fields; StringMap<size_t> fieldIds; StringMap<size_t> methods; bool isClass=false; bool isInterface=false; vector<size_t> interfaces; StringMap<size_t> requiredMethods; }; struct Program { vector<FunctionInfo> functions; StringMap<size_t> functionIds; vector<TypeInfo> types; StringMap<size_t> typeIds; vector<string> globals; StringMap<size_t> globalIds; vector<string> strings; StringMap<size_t> stringIds; vector<Value> compileConstants; StringMap<size_t> constantIds; vector<StaticType> globalTypes; StringMap<size_t> globalObjectTypes; StringMap<size_t> genericSpecializations; }; static size_t boundedEditDistance(string_view a, string_view b, size_t cutoff = 8) { if (a.size() > 128 || b.size() > 128) return cutoff + 1; // Three rows preserve adjacent transpositions in O(min(n,m)) workspace. if (a.size() < b.size()) swap(a, b); const size_t columns = b.size() + 1; vector<uint16_t> previous2(columns, 0), previous(columns, 0), current(columns, 0); for (size_t j = 0; j < columns; ++j) previous[j] = static_cast<uint16_t>(j); for (size_t i = 1; i <= a.size(); ++i) { current[0] = static_cast<uint16_t>(i); size_t rowMinimum = i; for (size_t j = 1; j < columns; ++j) { const size_t cost = lowerAscii(a[i - 1]) == lowerAscii(b[j - 1]) ? 0u : 1u; size_t value = min({ static_cast<size_t>(previous[j]) + 1, static_cast<size_t>(current[j - 1]) + 1, static_cast<size_t>(previous[j - 1]) + cost }); if (i > 1 && j > 1 && lowerAscii(a[i - 1]) == lowerAscii(b[j - 2]) && lowerAscii(a[i - 2]) == lowerAscii(b[j - 1])) { value = min(value, static_cast<size_t>(previous2[j - 2]) + 1); } current[j] = static_cast<uint16_t>(min<size_t>(value, 65535)); rowMinimum = min(rowMinimum, value); } previous2.swap(previous); previous.swap(current); if (rowMinimum > cutoff && i > b.size() + cutoff) return cutoff + 1; } return previous.back(); } static StringSet characterNgrams(const string&s){ StringSet out; const string x="^"+lowerAscii(s)+"$"; if(x.size()<3){out.insert(x);return out;} for(size_t i=0;i+3<=x.size();++i)out.insert(x.substr(i,3));return out; } static double setJaccard(const StringSet&a,const StringSet&b){ if(a.empty()&&b.empty()) return 1.0; const auto*small=&a; const auto*large=&b; if(small->size()>large->size()) swap(small,large); size_t n=0;for(const auto&x:*small)if(large->count(x))++n;const size_t u=a.size()+b.size()-n;return u?static_cast<double>(n)/static_cast<double>(u):0.0; } static string phoneticKey(const string&s){ string x;for(char c:s)if(isAsciiAlpha(u8(c)))x.push_back(lowerAscii(c));if(x.empty())return {}; auto code=[](char c)->char{switch(c){case 'b':case 'f':case 'p':case 'v':return '1';case 'c':case 'g':case 'j':case 'k':case 'q':case 's':case 'x':case 'z':return '2';case 'd':case 't':return '3';case 'l':return '4';case 'm':case 'n':return '5';case 'r':return '6';default:return '0';}}; string out(1,x[0]);char prev=code(x[0]);for(size_t i=1;i<x.size()&&out.size()<5;++i){char q=code(x[i]);if(q!='0'&&q!=prev)out.push_back(q);prev=q;}while(out.size()<5)out.push_back('0');return out; } // ------------------------------ Compiler --------------------------------- struct DiagnosticChoice { string word; double score=0.0; size_t edit=0; }; // The minimum score applies to ordinary expression/operator guesses. At an // explicit grammar boundary, the parser may request a low-confidence choice // list; such choices are always user-confirmed and are never auto-accepted. static constexpr double MIN_KEYWORD_SUGGESTION_SCORE = 0.10; static constexpr size_t MAX_KEYWORD_SUGGESTIONS = 5; class KeywordCorrector { public: KeywordCorrector(const DefinitionModel &definitions, int depth, bool disabled, bool automatic) : definitions_(definitions), depth_(depth), disabled_(disabled), automatic_(automatic) { profileCache_.reserve(64); } bool suggest(string &surface, const vector<string> &expected, size_t line, size_t column, bool allowLowConfidence = false, bool allowReservedMismatch = false) { if (disabled_ || surface.empty() || surface.size() > MAX_IDENTIFIER_BYTES || expected.empty()) return false; const string unknown = lowerAscii(surface); // A keyword that is already valid at this exact grammar position needs // no correction. Other reserved words remain eligible: e.g. "public" // can be a wrong keyword where "private" is expected. for (const string &candidate : expected) if (lowerAscii(candidate) == unknown) return false; if (!allowReservedMismatch && reservedKeywords().find(unknown) != reservedKeywords().end()) return false; // Cache reusable grammar-candidate profiles; keep the untrusted input // profile local so unique misspellings cannot grow the cache without bound. KeywordProfile unknownProfile; unknownProfile.fingerprint = definitions_.fingerprint(unknown, depth_); unknownProfile.ngrams = characterNgrams(unknown); unknownProfile.phonetic = phoneticKey(unknown); vector<DiagnosticChoice> ranked; ranked.reserve(expected.size()); for (const string &candidateText : expected) { const string candidate = lowerAscii(candidateText); if (candidate.empty() || candidate == unknown) continue; const KeywordProfile &candidateProfile = profileFor(candidate); const double definitionScore = definitions_.jaccard( unknownProfile.fingerprint, candidateProfile.fingerprint); const size_t edit = boundedEditDistance(unknown, candidate, 12); const size_t maximumLength = max(unknown.size(), candidate.size()); const double editScore = 1.0 - static_cast<double>( min<size_t>(edit, maximumLength)) / static_cast<double>(max<size_t>(1, maximumLength)); const double ngramScore = setJaccard( unknownProfile.ngrams, candidateProfile.ngrams); const double phoneticScore = (!unknownProfile.phonetic.empty() && unknownProfile.phonetic == candidateProfile.phonetic) ? 1.0 : 0.0; // Definition-expansion Jaccard remains the dominant signal. The // spelling signals resolve orthographic errors when dictionary // expansion has little or no overlap. const double score = 0.55 * definitionScore + 0.25 * editScore + 0.15 * ngramScore + 0.05 * phoneticScore; ranked.push_back({candidate, score, edit}); } if (ranked.empty()) return false; iterativeHeapSort(ranked.begin(), ranked.end(), [](const auto &a, const auto &b) { if (a.score != b.score) return a.score > b.score; if (a.edit != b.edit) return a.edit < b.edit; return a.word < b.word; }); if (ranked.front().score < MIN_KEYWORD_SUGGESTION_SCORE && !allowLowConfidence) return false; const size_t shown = min(MAX_KEYWORD_SUGGESTIONS, ranked.size()); cerr << "\nPossible keyword error at " << line << ":" << column << ": '" << surface << "'. Suggestion(s):"; for (size_t i = 0; i < shown; ++i) { cerr << "\n " << (i + 1) << ") " << ranked[i].word << " (score " << fixed << setprecision(3) << ranked[i].score << ")"; } const bool strong = ranked.front().score >= 0.88 && (shown == 1 || ranked.front().score - ranked[1].score >= 0.15); bool accept = automatic_ && strong; size_t selected = 0; if (!accept) { cerr << "\nReplace with '" << ranked.front().word << "'? [y/N"; if (shown > 1) cerr << " or 1-" << shown; cerr << "] " << flush; string answer; if (!getline(cin, answer)) return false; answer = lowerAscii(answer); if (answer == "y" || answer == "yes") { accept = true; } else if (!answer.empty()) { size_t choice = 0; bool numeric = true; for (char c : answer) { if (!isAsciiDigit(u8(c))) { numeric = false; break; } const size_t digit = static_cast<size_t>(c - '0'); if (choice > (numeric_limits<size_t>::max() - digit) / 10) { numeric = false; break; } choice = choice * 10 + digit; } if (numeric && choice >= 1 && choice <= shown) { selected = choice - 1; accept = true; } } } if (!accept) return false; surface = ranked[selected].word; return true; } private: struct KeywordProfile { StringSet fingerprint; StringSet ngrams; string phonetic; }; const DefinitionModel &definitions_; int depth_; bool disabled_, automatic_; StringMap<KeywordProfile> profileCache_; static const StringSet &reservedKeywords() { static const StringSet words = { "let","set","increase","decrease","display","read","if","while","then", "else","end","be","to","by","is","equal","not","less","greater","than", "or","and","plus","minus","times","true","false","generate","define", "function","method","class","structure","named","taking","field","create", "object","of","call","give","back","return","include","during","compilation", "constant","generic","for","type","that","inherits","from","implements", "interface","require","module","a","an","this","public","private","protected", "try","catch","finally","throw","at","length","append","divided","modulo", "use","import","as","float","number","map","dictionary" }; return words; } const KeywordProfile &profileFor(const string &word) { const auto found = profileCache_.find(word); if (found != profileCache_.end()) return found->second; KeywordProfile profile; profile.fingerprint = definitions_.fingerprint(word, depth_); profile.ngrams = characterNgrams(word); profile.phonetic = phoneticKey(word); return profileCache_.emplace(word, move(profile)).first->second; } }; class Compiler { public: Compiler(vector<Token> tokens, KeywordCorrector &corrector) : tokens_(move(tokens)), corrector_(corrector) { FunctionInfo main; main.name="$main"; program_.functions.push_back(move(main)); } Program compile(){ while(!isEnd()){ const size_t before=pos_; try { correctBoundaryKeywords(); if(activeFunction_!=0 && isWord("end") && (isWordAt(1,"function")||isWordAt(1,"method"))){closeFunction();continue;} if(currentType_!=NO_ID && activeFunction_==0 && isWord("end") && (isWordAt(1,"class")||isWordAt(1,"structure")||isWordAt(1,"interface"))){closeType();continue;} if(isWord("end") && (isWordAt(1,"if")||isWordAt(1,"while")||isWordAt(1,"try"))){ const bool matches=!controls_.empty()&&((controls_.back().kind==Control::If&&isWordAt(1,"if"))||(controls_.back().kind==Control::While&&isWordAt(1,"while"))||(controls_.back().kind==Control::Try&&isWordAt(1,"try"))); if(matches)closeControl();else {pos_+=2;optionalSemicolon();} continue; } if(isWord("else")){handleElse();continue;} if(isWord("catch")){handleCatch();continue;} if(isWord("finally")){handleFinally();continue;} if(isWord("try")){parseTry();continue;} if(activeFunction_==0 && currentType_==NO_ID && isWord("module")){parseModule();continue;} if(activeFunction_==0 && currentType_==NO_ID && isWord("use")){parseUseModule();continue;} if(activeFunction_==0 && currentType_==NO_ID && isWord("end") && isWordAt(1,"module")){closeModule();continue;} if(activeFunction_==0 && isWord("define")){parseDefinition();continue;} if(activeFunction_==0 && currentType_!=NO_ID && isWord("require")){parseInterfaceRequirement();continue;} if(activeFunction_==0 && currentType_!=NO_ID && (isWord("public")||isWord("private")||isWord("protected"))){ currentAccess_=isWord("public")?AccessLevel::Public:(isWord("protected")?AccessLevel::Protected:AccessLevel::Private);++pos_; if (!isWord("field") && !isWord("define")) (void)suggestKeyword({"field", "define"}); if(isWord("field")){parseField();continue;} if(isWord("define")){parseDefinition();continue;} if (peek().kind == TokenKind::Word && (isWordAt(1, "be") || isSymbolAt(pos_ + 1, ";"))) { parseField(false); continue; } error("access modifier must precede a field or method definition"); } if(activeFunction_==0 && currentType_!=NO_ID && isWord("field")){parseField();continue;} if(activeFunction_==0 && isWord("during")){parseCompileTime();continue;} if(isWord("include"))error("include directives must occupy a complete source line and are expanded before compilation"); if(activeFunction_==0 && currentType_!=NO_ID && isWord("define")){parseDefinition();continue;} parseStatement(); } catch(const exception &ex) { if(diagnostics_.size()<MAX_DIAGNOSTICS)diagnostics_.push_back(ex.what()); synchronize(before); } if(pos_==before&&!isEnd())++pos_; } if(activeFunction_!=0)diagnostics_.push_back("unterminated function or method definition"); if(currentType_!=NO_ID)diagnostics_.push_back("unterminated class, structure, or interface definition"); if(!controls_.empty())diagnostics_.push_back("unterminated control block"); if(!currentModule_.empty())diagnostics_.push_back("unterminated module '"+currentModule_+"'"); if(!diagnostics_.empty()){ cerr<<"EnglishLike compiler diagnostics ("<<diagnostics_.size()<<"):\n"; for(size_t i=0;i<diagnostics_.size();++i)cerr<<" "<<(i+1)<<". "<<diagnostics_[i]<<'\n'; throw runtime_error("compilation failed"); } return move(program_); } void synchronize(size_t before){ if(pos_==before&&!isEnd())++pos_; while(!isEnd()){ if(matchSymbol(";"))return; if(isWord("end")&&(isWordAt(1,"if")||isWordAt(1,"while")||isWordAt(1,"try"))){ bool matches=false; if(!controls_.empty()){ matches=(controls_.back().kind==Control::If&&isWordAt(1,"if"))|| (controls_.back().kind==Control::While&&isWordAt(1,"while"))|| (controls_.back().kind==Control::Try&&isWordAt(1,"try")); } if(matches)return; pos_+=2;(void)matchSymbol(";");continue; } if(isWord("end")||isWord("else")||isWord("catch")||isWord("finally")){ if((isWord("else")&&!controls_.empty()&&controls_.back().kind==Control::If)|| ((isWord("catch")||isWord("finally"))&&!controls_.empty()&&controls_.back().kind==Control::Try))return; ++pos_;continue; } if(isWord("let")||isWord("set")||isWord("display")||isWord("read")||isWord("if")||isWord("while")||isWord("define")||isWord("create")||isWord("throw")||isWord("append"))return; ++pos_; } } private: static constexpr size_t NO_ID=numeric_limits<size_t>::max(); struct Control { enum Kind{If,While,Try} kind; size_t branch=0,loopStart=0,endJump=NO_ID; bool hasElse=false,hasFinally=false; }; struct OpEntry { string op; int precedence=0; bool unary=false; bool paren=false; bool call=false; size_t commas=0; char closing=')'; }; vector<Token> tokens_; KeywordCorrector &corrector_; size_t pos_=0,activeFunction_=0,currentType_=NO_ID; Program program_; vector<Control> controls_; string currentModule_; vector<string> moduleStack_; vector<string> importedModules_; AccessLevel currentAccess_=AccessLevel::Public; vector<string> diagnostics_; size_t generatedSourceBytes_=0; StaticType lastExpressionType_=StaticType::Unknown; const Token&peek(size_t n=0)const{return tokens_[min(pos_+n,tokens_.size()-1)];} bool isEnd()const{return peek().kind==TokenKind::End;} bool isWord(const string&w)const{return peek().kind==TokenKind::Word&&lowerAscii(peek().text)==w;} bool isWordAt(size_t n,const string&w)const{return peek(n).kind==TokenKind::Word&&lowerAscii(peek(n).text)==w;} bool isSymbol(const string&s)const{return peek().kind==TokenKind::Symbol&&peek().text==s;} bool matchWord(const string&w){if(!isWord(w))return false;++pos_;return true;} bool matchSymbol(const string&s){if(!isSymbol(s))return false;++pos_;return true;} string identifier(){if(peek().kind!=TokenKind::Word)error("expected an identifier");string n=lowerAscii(peek().text);++pos_;return n;} [[noreturn]] void error(const string&m)const{const Token&t=peek();throw runtime_error("compiler error at "+to_string(t.line)+":"+to_string(t.column)+": "+m+(t.kind==TokenKind::End?" (end of file)":" near '"+t.text+"'"));} [[noreturn]] void errorAt(const Token&t,const string&m)const{throw runtime_error("compiler error at "+to_string(t.line)+":"+to_string(t.column)+": "+m+(t.kind==TokenKind::End?" (end of file)":" near '"+t.text+"'"));} size_t addString(const string&s){auto f=program_.stringIds.find(s);if(f!=program_.stringIds.end())return f->second;size_t id=program_.strings.size();program_.strings.push_back(s);program_.stringIds.emplace(s,id);return id;} void emit(Op op,size_t a=0,size_t b=0){auto&c=program_.functions[activeFunction_].code;if(c.size()>10000000)error("function instruction limit exceeded");c.push_back({op,a,b});} void patch(size_t at,size_t target){auto&c=program_.functions[activeFunction_].code;if(at>=c.size())error("invalid compiler jump patch");c[at].a=target;} void optionalSemicolon(){(void)matchSymbol(";");} bool suggestKeyword(const vector<string> &expected, bool allowLowConfidence = false, bool allowReservedMismatch = false) { if (peek().kind != TokenKind::Word) return false; return corrector_.suggest(tokens_[pos_].text, expected, peek().line, peek().column, allowLowConfidence, allowReservedMismatch); } bool suggestKeywordAt(size_t index, const vector<string> &expected, bool allowLowConfidence = false, bool allowReservedMismatch = false) { if (index >= tokens_.size() || tokens_[index].kind != TokenKind::Word) return false; return corrector_.suggest(tokens_[index].text, expected, tokens_[index].line, tokens_[index].column, allowLowConfidence, allowReservedMismatch); } bool matchOneOf(const vector<string> &expected, string &matched) { for (const string &word : expected) { if (matchWord(word)) { matched = word; return true; } } if (suggestKeyword(expected, true, true)) { for (const string &word : expected) { if (matchWord(word)) { matched = word; return true; } } } return false; } void expectWord(const string &word, const vector<string> &expected = {}) { if (matchWord(word)) return; if (!expected.empty() && suggestKeyword(expected, true, true) && matchWord(word)) return; error("expected keyword '" + word + "'"); } void correctBoundaryKeywords() { // Clause keywords take precedence over generic terminator correction. // This matters for short typos such as "els", which is closer to // "else" than to the otherwise legal block terminator "end". if (!isWord("end") && !controls_.empty()) { const Control &control = controls_.back(); if (control.kind == Control::If && !control.hasElse) { static const vector<string> elseClause = {"else"}; (void)suggestKeyword(elseClause); } else if (control.kind == Control::Try) { static const vector<string> catchClause = {"catch"}; static const vector<string> finallyClause = {"finally"}; if (!control.hasElse) (void)suggestKeyword(catchClause); else if (!control.hasFinally) (void)suggestKeyword(finallyClause); } } const bool mayEnd = activeFunction_ != 0 || currentType_ != NO_ID || !controls_.empty() || !moduleStack_.empty(); if (mayEnd && !isWord("end")) { static const vector<string> endKeyword = {"end"}; (void)suggestKeyword(endKeyword); } // Correct the second word only after the first word is known to be // "end". The expected suffix is determined by the open grammar frame. if (isWord("end")) { if (!controls_.empty()) { static const vector<string> ifEnd = {"if"}; static const vector<string> whileEnd = {"while"}; static const vector<string> tryEnd = {"try"}; const Control::Kind kind = controls_.back().kind; (void)suggestKeywordAt(pos_ + 1, kind == Control::If ? ifEnd : (kind == Control::While ? whileEnd : tryEnd), true, true); } else if (activeFunction_ != 0) { static const vector<string> functionEnd = {"function"}; static const vector<string> methodEnd = {"method"}; const bool method = program_.functions[activeFunction_].ownerType != NO_ID; (void)suggestKeywordAt(pos_ + 1, method ? methodEnd : functionEnd, true, true); } else if (currentType_ != NO_ID) { static const vector<string> classEnd = {"class"}; static const vector<string> structureEnd = {"structure"}; static const vector<string> interfaceEnd = {"interface"}; const TypeInfo &type = program_.types[currentType_]; (void)suggestKeywordAt(pos_ + 1, type.isInterface ? interfaceEnd : (type.isClass ? classEnd : structureEnd), true, true); } else if (!moduleStack_.empty()) { static const vector<string> moduleEnd = {"module"}; (void)suggestKeywordAt(pos_ + 1, moduleEnd, true, true); } } if (activeFunction_ == 0 && currentType_ == NO_ID) { static const vector<string> topLevel = { "define", "module", "use", "include", "during", "let", "set", "increase", "decrease", "display", "read", "if", "while", "try", "create", "call", "append", "throw", "give", "return" }; (void)suggestKeyword(topLevel, true); } else if (activeFunction_ == 0 && currentType_ != NO_ID) { static const vector<string> typeBody = { "public", "private", "protected", "field", "define", "require", "during" }; (void)suggestKeyword(typeBody, true); } } size_t declareVariable(const string&name){ auto&fn=program_.functions[activeFunction_];if(fn.localIds.count(name))error("duplicate local variable '"+name+"'"); if (fn.localNames.size() >= MAX_LOCALS_PER_FUNCTION) error("function local-variable limit exceeded"); const size_t id=fn.localNames.size();fn.localNames.push_back(name);fn.localIds.emplace(name,id);fn.localTypes.push_back(StaticType::Unknown);return id; } struct VarRef{bool global=false;size_t id=0;}; VarRef resolveVariable(const string&name)const{ const auto&fn=program_.functions[activeFunction_];auto l=fn.localIds.find(name);if(l!=fn.localIds.end())return {false,l->second}; auto g=program_.globalIds.find(name);if(g!=program_.globalIds.end())return {true,g->second}; throw runtime_error("unknown identifier '"+name+"' (identifiers are not keyword-correction candidates)"); } void emitLoad(const VarRef&r){emit(r.global?Op::LoadGlobal:Op::LoadLocal,r.id);} void emitStore(const VarRef&r){emit(r.global?Op::StoreGlobal:Op::StoreLocal,r.id);} StaticType staticTypeOf(const Value &value) const { if(holds_alternative<BigNat>(value))return StaticType::Natural; if(holds_alternative<double>(value))return StaticType::Floating; if(holds_alternative<bool>(value))return StaticType::Boolean; if(holds_alternative<string>(value))return StaticType::String; return StaticType::Object; } size_t receiverType(const string &name) const { if(name=="this")return currentType_; const FunctionInfo &function=program_.functions[activeFunction_]; auto local=function.localObjectTypes.find(name);if(local!=function.localObjectTypes.end())return local->second; auto global=program_.globalObjectTypes.find(name);if(global!=program_.globalObjectTypes.end())return global->second; return NO_ID; } const FieldInfo *fieldFor(size_t typeId,const string &name) const { if(typeId>=program_.types.size())return nullptr; const TypeInfo &type=program_.types[typeId];auto found=type.fieldIds.find(name); return found==type.fieldIds.end()?nullptr:&type.fields[found->second]; } size_t methodFor(size_t typeId,const string &name) const { size_t hops=0; while(typeId<program_.types.size()&&hops++<=program_.types.size()){ const TypeInfo &type=program_.types[typeId];auto found=type.methods.find(name);if(found!=type.methods.end())return found->second; if(type.base==NO_ID)break; typeId=type.base; } return NO_ID; } bool accessAllowed(AccessLevel access,size_t declaringType) const { if(access==AccessLevel::Public)return true; if(currentType_==declaringType)return true; if(access==AccessLevel::Private)return false; size_t type=currentType_,hops=0; while(type<program_.types.size()&&hops++<=program_.types.size()){ if(type==declaringType)return true; type=program_.types[type].base; } return false; } StaticType variableStaticType(const string &name) const { if(name=="this")return StaticType::Object; const FunctionInfo &function=program_.functions[activeFunction_];auto local=function.localIds.find(name); if(local!=function.localIds.end()&&local->second<function.localTypes.size())return function.localTypes[local->second]; auto global=program_.globalIds.find(name);if(global!=program_.globalIds.end()&&global->second<program_.globalTypes.size())return program_.globalTypes[global->second]; return StaticType::Unknown; } string resolveFunctionName(const string &name) const { if(program_.functionIds.count(name))return name; if(!currentModule_.empty()&&program_.functionIds.count(currentModule_+"::"+name))return currentModule_+"::"+name; for(const string &module:importedModules_){const string candidate=module+"::"+name;if(program_.functionIds.count(candidate))return candidate;} return name; } size_t declareGlobal(const string&name){if(program_.globalIds.count(name))error("duplicate global variable '"+name+"'");if(program_.globals.size()>=MAX_GLOBALS)error("global-variable limit exceeded");size_t id=program_.globals.size();program_.globals.push_back(name);program_.globalIds.emplace(name,id);program_.globalTypes.push_back(StaticType::Unknown);return id;} void expectSemicolon(){if(!matchSymbol(";"))error("expected ';'");} bool stopExpression(const vector<string>&stops)const{ if(peek().kind==TokenKind::End||isSymbol(";")||isSymbol(":"))return true; if(peek().kind==TokenKind::Word){const string w=lowerAscii(peek().text);return find(stops.begin(),stops.end(),w)!=stops.end();}return false; } int precedence(const string&op)const{ if(op=="or") return 1; if(op=="and") return 2; if(op=="=="||op=="!="||op=="<"||op=="<="||op==">"||op==">=") return 3; if(op=="+"||op=="-") return 4; if(op=="*") return 5; if(op=="/") return 5; if(op=="%") return 5; if(op=="at" || op=="length") return 7; if(op=="not") return 6; return -1; } void emitOperator(const string&op){ if(op=="+")emit(Op::Add);else if(op=="-")emit(Op::Subtract);else if(op=="*")emit(Op::Multiply); else if(op=="==")emit(Op::Equal);else if(op=="!=")emit(Op::NotEqual);else if(op=="<")emit(Op::Less);else if(op=="<=")emit(Op::LessEqual);else if(op==">")emit(Op::Greater);else if(op==">=")emit(Op::GreaterEqual); else if(op=="/")emit(Op::Divide);else if(op=="%")emit(Op::Remainder); else if(op=="at")emit(Op::ArrayGet);else if(op=="length")emit(Op::ArrayLength); else if(op=="and")emit(Op::BooleanAnd);else if(op=="or")emit(Op::BooleanOr);else if(op=="not")emit(Op::BooleanNot);else error("unsupported operator '"+op+"'"); } StaticType emitVariableOrConstant(const string&name){ if(name=="this"){const auto&fn=program_.functions[activeFunction_];if(fn.ownerType==NO_ID)error("'this' is only valid inside a method");emit(Op::LoadThis);return StaticType::Object;} auto c=program_.constantIds.find(name);if(c!=program_.constantIds.end()){ const Value&v=program_.compileConstants[c->second];const StaticType type=staticTypeOf(v); if(holds_alternative<BigNat>(v)||holds_alternative<double>(v)){size_t k=program_.compileConstants.size();program_.compileConstants.push_back(v);emit(Op::PushNumber,k);} else if(holds_alternative<bool>(v)){emit(Op::PushBoolean,get<bool>(v)?1:0);} else if(holds_alternative<string>(v))emit(Op::PushString,addString(get<string>(v))); else error("object values cannot be compile-time constants"); return type; } const VarRef r=resolveVariable(name);emitLoad(r);return variableStaticType(name); } string specializeGeneric(const string &name,const vector<StaticType> &argumentTypes){ auto found=program_.functionIds.find(name);if(found==program_.functionIds.end())return name; const size_t baseId=found->second;const FunctionInfo &base=program_.functions[baseId];if(!base.generic)return name; string key=to_string(baseId);for(StaticType type:argumentTypes){key.push_back(':');key+=to_string(static_cast<unsigned int>(type));} auto cached=program_.genericSpecializations.find(key);if(cached!=program_.genericSpecializations.end())return program_.functions[cached->second].name; if(program_.functions.size()>=MAX_FUNCTIONS)error("generic specialization-count limit exceeded"); FunctionInfo specialized=base;specialized.generic=false;specialized.name=base.name+"$specialized$"+key; for(size_t i=0;i<specialized.parameterTypeNames.size()&&i<argumentTypes.size();++i){ if(find(base.genericParameters.begin(),base.genericParameters.end(),specialized.parameterTypeNames[i])!=base.genericParameters.end()){ specialized.parameterTypes[i]=argumentTypes[i]; if(i<specialized.localTypes.size())specialized.localTypes[i]=argumentTypes[i]; } } const size_t id=program_.functions.size();program_.functions.push_back(move(specialized));program_.functionIds.emplace(program_.functions[id].name,id);program_.genericSpecializations.emplace(move(key),id); return program_.functions[id].name; } void parseExpression(const vector<string>&stops={}){ vector<OpEntry> ops;ops.reserve(32);bool expecting=true;size_t produced=0;size_t iterations=0; vector<StaticType> inferred;inferred.reserve(32); auto applyOperator=[&](const string &op){ emitOperator(op); if(op=="not"){if(!inferred.empty())inferred.back()=StaticType::Boolean;return;} if(op=="length"){if(!inferred.empty()){if(inferred.back()!=StaticType::Array&&inferred.back()!=StaticType::Map&&inferred.back()!=StaticType::Unknown)error("length requires an array or map collection");inferred.back()=StaticType::Natural;}return;} if(inferred.size()<2)return; const StaticType right=inferred.back();inferred.pop_back();const StaticType left=inferred.back();inferred.pop_back(); StaticType result=StaticType::Unknown; if(op=="at"){ if(left!=StaticType::Array&&left!=StaticType::Map&&left!=StaticType::Unknown)error("indexed access requires an array or map"); if(left==StaticType::Array&&right!=StaticType::Natural&&right!=StaticType::Unknown)error("array index must be a natural number"); if(left==StaticType::Map&&right!=StaticType::String&&right!=StaticType::Unknown)error("map keys must be strings"); } else if(op=="and"||op=="or"||op=="=="||op=="!="||op=="<"||op=="<="||op==">"||op==">=")result=StaticType::Boolean; else if(op=="+"&&left==StaticType::String&&right==StaticType::String)result=StaticType::String; else if((left==StaticType::Natural||left==StaticType::Floating||left==StaticType::Unknown)&&(right==StaticType::Natural||right==StaticType::Floating||right==StaticType::Unknown))result=(op=="/"||left==StaticType::Floating||right==StaticType::Floating)?StaticType::Floating:(left==StaticType::Natural&&right==StaticType::Natural?StaticType::Natural:StaticType::Unknown); else if(left!=StaticType::Unknown&&right!=StaticType::Unknown)error("operator '"+op+"' is not defined for these static types"); if(op=="at")result=StaticType::Unknown; inferred.push_back(result); }; auto popOps=[&](int p){while(!ops.empty()&&!ops.back().paren&&precedence(ops.back().op)>=p){applyOperator(ops.back().op);ops.pop_back();}}; while(!stopExpression(stops)){ if (!expecting && !stops.empty() && peek().kind == TokenKind::Word && suggestKeyword(stops) && stopExpression(stops)) break; const bool hasParen=any_of(ops.rbegin(),ops.rend(),[](const OpEntry&o){return o.paren;}); if(!hasParen&&(isSymbol(",")||isSymbol(")")))break; if(++iterations>MAX_TOKENS)error("expression token limit exceeded"); const Token&t=peek(); if(t.kind==TokenKind::Number){ if(!expecting)error("missing operator before number"); const size_t id=program_.compileConstants.size(); if(t.text.find('.')!=string::npos)program_.compileConstants.emplace_back(parseFiniteFloat(t.text)); else program_.compileConstants.emplace_back(BigNat(t.text)); emit(Op::PushNumber,id);inferred.push_back(t.text.find('.')!=string::npos?StaticType::Floating:StaticType::Natural);++pos_;expecting=false;++produced;continue; } if(t.kind==TokenKind::String){if(!expecting)error("missing operator before string");emit(Op::PushString,addString(t.text));inferred.push_back(StaticType::String);++pos_;expecting=false;++produced;continue;} if(t.kind==TokenKind::Word){ const string w=lowerAscii(t.text); if (expecting && (w == "true" || w == "false")) { emit(Op::PushBoolean, w == "true" ? 1 : 0);inferred.push_back(StaticType::Boolean); ++pos_; expecting = false; ++produced; continue; } if(w=="not"&&expecting){ops.push_back({"not",6,true,false,false,0});++pos_;continue;} if(w=="length"&&expecting&&isWordAt(1,"of")){ops.push_back({"length",7,true,false,false,0});pos_+=2;continue;} if(w=="map"&&expecting&&isSymbolAt(pos_+1,"(")){++pos_;++pos_;ops.push_back({"",0,false,true,true,0,')'});callNames_.push_back("$map");callNameDepth_.push_back(ops.size());continue;} string op; if(w=="plus")op="+";else if(w=="minus")op="-";else if(w=="times")op="*";else if(w=="divided"&&isWordAt(1,"by")){op="/";pos_+=2;}else if(w=="modulo")op="%";else if(w=="at")op="at";else if(w=="and")op="and";else if(w=="or")op="or"; else if(w=="is"){ if(isWordAt(1,"less")){op="<";pos_+=2;expectWord("than",{"than"});if(matchWord("or")){expectWord("equal",{"equal"});expectWord("to",{"to"});op="<=";}} else if(isWordAt(1,"greater")){op=">";pos_+=2;expectWord("than",{"than"});if(matchWord("or")){expectWord("equal",{"equal"});expectWord("to",{"to"});op=">=";}} else if(isWordAt(1,"not")){op="!=";pos_+=2;expectWord("equal",{"equal"});(void)matchWord("to");} else if(isWordAt(1,"equal")){op="==";pos_+=2;(void)matchWord("to");} else { ++pos_; const bool repaired=suggestKeyword({"equal","less","greater","not"}, true, true); --pos_; if(!repaired)error("expected equal, less than, greater than, or not equal after 'is'"); if(isWordAt(1,"less")){op="<";pos_+=2;expectWord("than",{"than"});if(matchWord("or")){expectWord("equal",{"equal"});expectWord("to",{"to"});op="<=";}} else if(isWordAt(1,"greater")){op=">";pos_+=2;expectWord("than",{"than"});if(matchWord("or")){expectWord("equal",{"equal"});expectWord("to",{"to"});op=">=";}} else if(isWordAt(1,"not")){op="!=";pos_+=2;expectWord("equal",{"equal"});(void)matchWord("to");} else if(isWordAt(1,"equal")){op="==";pos_+=2;(void)matchWord("to");} } } else if (w == "less" || w == "greater") { if (!isWordAt(1, "than")) (void)suggestKeywordAt(pos_ + 1, {"than"}, true, true); if (isWordAt(1, "than")) { op = w == "less" ? "<" : ">"; pos_ += 2; } } if(!op.empty()){ if(expecting) error("operator has no left operand"); const int p=precedence(op); popOps(p); ops.push_back({op,p,false,false,false,0}); if(w!="is"&&w!="less"&&w!="greater"&&w!="divided") ++pos_; expecting=true; continue; } if(!expecting){ static const vector<string>operators={ "plus","minus","times","divided","modulo","and","or","is","less","greater" }; if(suggestKeyword(operators))continue; error("missing operator before identifier"); } if(isSymbolAt(pos_+1,"(")){ const string fname=resolveFunctionName(w);++pos_;++pos_;ops.push_back({"",0,false,true,true,0,')'}); callNames_.push_back(fname);callNameDepth_.push_back(ops.size());continue; } if(receiverType(w)>=program_.types.size()&&peek(1).kind==TokenKind::Word){ string qualified=w;size_t scan=pos_+1,matchedEnd=0;string matchedName; while(scan<tokens_.size()&&tokens_[scan].kind==TokenKind::Word){ qualified+="::"+lowerAscii(tokens_[scan].text); if(tokens_[scan+1].kind==TokenKind::Symbol&&tokens_[scan+1].text=="("&&program_.functionIds.count(qualified)){matchedEnd=scan+1;matchedName=qualified;} ++scan; } if(matchedEnd){pos_=matchedEnd+1;ops.push_back({"",0,false,true,true,0,')'});callNames_.push_back(matchedName);callNameDepth_.push_back(ops.size());continue;} } // Dot-free member syntax: this balance, customer id, customer deposit(...). const size_t receiver=receiverType(w);StaticType valueType=emitVariableOrConstant(w);++pos_; if(peek().kind==TokenKind::Word && isSymbolAt(pos_+1,"(") && receiver<program_.types.size()){ const string member=lowerAscii(peek().text);const size_t methodId=methodFor(receiver,member); if(methodId!=NO_ID){if(!accessAllowed(program_.functions[methodId].access,program_.functions[methodId].ownerType))error("method '"+member+"' is not accessible here");++pos_;++pos_;ops.push_back({"",0,false,true,true,0,')'});callNames_.push_back("method:"+member+":"+to_string(receiver));callNameDepth_.push_back(ops.size());continue;} } if(peek().kind==TokenKind::Word && receiver<program_.types.size()){ const string member=lowerAscii(peek().text);const FieldInfo *field=fieldFor(receiver,member); if(field){if(!accessAllowed(field->access,field->declaringType))error("field '"+member+"' is not accessible here");++pos_;emit(Op::GetField,addString(member));valueType=field->type;} } inferred.push_back(valueType);expecting=false;++produced;continue; } if(t.kind==TokenKind::Symbol){ if(t.text=="(") {if(!expecting)error("missing operator before '('");++pos_;ops.push_back({"",0,false,true,false,0,')'});continue;} if(t.text=="[") {if(!expecting)error("missing operator before '['");++pos_;ops.push_back({"",0,false,true,true,0,']'});callNames_.push_back("$array");callNameDepth_.push_back(ops.size());continue;} if(t.text==")"||t.text=="]"){ if(ops.empty()||none_of(ops.rbegin(),ops.rend(),[](const OpEntry&o){return o.paren;}))break; while(!ops.empty()&&!ops.back().paren){emitOperator(ops.back().op);ops.pop_back();} if(ops.empty()) error("unmatched ')'"); OpEntry frame=ops.back();if(frame.closing!=t.text[0])error("mismatched expression delimiter");ops.pop_back(); ++pos_; if(frame.call){ if(expecting&&frame.commas>0)error("missing function argument"); const size_t argc=frame.commas+(expecting?0:1); if(!callNames_.empty()){ const string fn=callNames_.back();callNames_.pop_back();if(!callNameDepth_.empty())callNameDepth_.pop_back(); StaticType result=StaticType::Unknown; vector<StaticType> argumentTypes(argc,StaticType::Unknown); for(size_t a=argc;a>0;--a){if(!inferred.empty()){argumentTypes[a-1]=inferred.back();inferred.pop_back();}} if(fn=="$array"){emit(Op::NewArray,0,argc);result=StaticType::Array;} else if(fn=="$map") {if(argc%2!=0)error("map constructor requires alternating string keys and values");emit(Op::NewMap,0,argc);result=StaticType::Map;} else if(fn.rfind("method:",0)==0){ const size_t split=fn.rfind(':'); const string method=fn.substr(7,split-7); const size_t receiver=static_cast<size_t>(stoull(fn.substr(split+1))); const size_t methodId=methodFor(receiver,method); if(methodId==NO_ID)error("method resolution failed for '"+method+"'"); const FunctionInfo &called=program_.functions[methodId]; if(called.parameters.size()!=argc)error("method '"+method+"' expects "+to_string(called.parameters.size())+" arguments"); for(size_t a=0;a<argumentTypes.size()&&a<called.parameterTypes.size();++a){ const StaticType expected=called.parameterTypes[a],actual=argumentTypes[a]; if(expected!=StaticType::Unknown&&actual!=StaticType::Unknown&&expected!=actual&&!(expected==StaticType::Floating&&actual==StaticType::Natural))error("argument "+to_string(a+1)+" of method '"+method+"' has an incompatible inferred type"); } result=called.returnType;emit(Op::CallMethod,addString(method),argc); } else { const string targetName=specializeGeneric(fn,argumentTypes);auto target=program_.functionIds.find(fn); if(target!=program_.functionIds.end()){ const FunctionInfo &called=program_.functions[target->second]; for(size_t a=0;a<argumentTypes.size()&&a<called.parameterTypes.size();++a){const StaticType expected=called.parameterTypes[a],actual=argumentTypes[a];if(expected!=StaticType::Unknown&&actual!=StaticType::Unknown&&expected!=actual&&!(expected==StaticType::Floating&&actual==StaticType::Natural))error("argument "+to_string(a+1)+" of '"+fn+"' has an incompatible inferred type");} result=called.returnType; } emit(Op::Call,addString(targetName),argc); } inferred.push_back(result); } } else if(!expecting&&inferred.empty())inferred.push_back(StaticType::Unknown); expecting=false;++produced;continue; } if(t.text==","){ auto marker=find_if(ops.rbegin(),ops.rend(),[](const OpEntry&o){return o.paren;}); if(marker==ops.rend()||!marker->call||expecting)error("unexpected comma; commas are only valid between function arguments"); while(!ops.empty()&&!ops.back().paren){emitOperator(ops.back().op);ops.pop_back();} ++ops.back().commas;++pos_;expecting=true;continue; } string op=t.text;if(op=="&&")op="and";if(op=="||")op="or"; if(op=="!"&&expecting){ops.push_back({"not",6,true,false,false,0});++pos_;continue;} if(op=="+"||op=="-"||op=="*"||op=="/"||op=="%"||op=="=="||op=="!="||op=="<"||op=="<="||op==">"||op==">="||op=="and"||op=="or"){ if(expecting) error("operator has no left operand"); int p=precedence(op); popOps(p); ops.push_back({op,p,false,false,false,0}); ++pos_; expecting=true; continue; } } break; } if(produced==0||expecting)error("expected an expression"); while(!ops.empty()){if(ops.back().paren)error("unclosed parenthesis");applyOperator(ops.back().op);ops.pop_back();} lastExpressionType_=inferred.empty()?StaticType::Unknown:inferred.back(); callNames_.clear();callNameDepth_.clear(); } vector<string> callNames_;vector<size_t> callNameDepth_; bool isSymbolAt(size_t at,const string&s)const{return tokens_[min(at,tokens_.size()-1)].kind==TokenKind::Symbol&&tokens_[min(at,tokens_.size()-1)].text==s;} string qualify(const string &name) const {return currentModule_.empty()?name:currentModule_+"::"+name;} size_t resolveType(const string &name) const { auto exact=program_.typeIds.find(name);if(exact!=program_.typeIds.end())return exact->second; if(!currentModule_.empty()){auto local=program_.typeIds.find(currentModule_+"::"+name);if(local!=program_.typeIds.end())return local->second;} for(const string &module:importedModules_){auto imported=program_.typeIds.find(module+"::"+name);if(imported!=program_.typeIds.end())return imported->second;} return NO_ID; } void parseModule(){ expectWord("module",{"module"}); moduleStack_.push_back(identifier()); currentModule_.clear(); for(const string &part:moduleStack_){if(!currentModule_.empty())currentModule_+="::";currentModule_+=part;} if(!matchSymbol(":"))error("expected ':' after module name"); } void closeModule(){ expectWord("end",{"end"});expectWord("module",{"module"}); if(moduleStack_.empty())error("'end module' without an open module"); moduleStack_.pop_back();currentModule_.clear(); for(const string &part:moduleStack_){if(!currentModule_.empty())currentModule_+="::";currentModule_+=part;} optionalSemicolon(); } void parseUseModule(){ expectWord("use",{"use"});expectWord("module",{"module"}); if(peek().kind!=TokenKind::Word)error("expected module name"); string name=identifier(); while(peek().kind==TokenKind::Word){name+="::"+identifier();} if(find(importedModules_.begin(),importedModules_.end(),name)==importedModules_.end())importedModules_.push_back(name); expectSemicolon(); } void parseInterfaceRequirement(){ expectWord("require",{"require"});expectWord("method",{"method"});expectWord("named",{"named"}); const string name=identifier();size_t count=0; if(matchWord("taking")){ if(matchSymbol("(")){if(!matchSymbol(")")){for(;;){(void)identifier();++count;if(matchSymbol(")"))break;if(!matchSymbol(","))error("expected ',' or ')' in interface method signature");}}} else {for(;;){(void)identifier();++count;if(!matchWord("and")&&!matchSymbol(","))break;}} } if(currentType_==NO_ID||!program_.types[currentType_].isInterface)error("method requirements are valid only inside an interface"); if(!program_.types[currentType_].requiredMethods.emplace(name,count).second)error("duplicate interface method requirement '"+name+"'"); expectSemicolon(); } StaticType parseTypeAnnotation(){ if(peek().kind!=TokenKind::Word)error("expected a type name after 'as'"); const string type=lowerAscii(peek().text);++pos_; if(type=="number"||type=="integer"||type=="natural")return StaticType::Natural; if(type=="float"||type=="floating")return StaticType::Floating; if(type=="string"||type=="text")return StaticType::String; if(type=="boolean"||type=="bool")return StaticType::Boolean; if(type=="array"||type=="list"||type=="collection")return StaticType::Array; if(type=="map"||type=="dictionary")return StaticType::Map; if(type=="object"||resolveType(type)!=NO_ID)return StaticType::Object; return StaticType::Unknown; } void parseDefinition(){ expectWord("define",{"define"});if(!matchWord("a"))(void)matchWord("an"); bool generic=matchWord("generic"); string declarationKind; if(matchOneOf({"interface", "class", "structure", "function", "method"}, declarationKind) && declarationKind != "function" && declarationKind != "method"){ const bool isInterface = declarationKind == "interface"; const bool isClass = declarationKind == "class"; expectWord("named",{"named"});const string shortTypeName=identifier();const string name=qualify(shortTypeName); if(program_.typeIds.count(name))error("type already defined: '"+name+"'"); if(program_.types.size()>=MAX_TYPES)error("type-count limit exceeded"); TypeInfo type;type.name=name;type.isClass=isClass;type.isInterface=isInterface; if (matchWord("that") || (peek().kind == TokenKind::Word && !isWord("for") && suggestKeyword({"that"}, true, true) && matchWord("that"))) { string inheritanceKind; if (matchOneOf({"inherits", "implements"}, inheritanceKind) && inheritanceKind == "inherits") { expectWord("from",{"from"});const string baseName=identifier();const size_t base=resolveType(baseName);if(base==NO_ID)error("base type must be defined before derived type: '"+baseName+"'"); if(program_.types[base].isInterface) error("a class cannot inherit implementation from an interface"); type.base=base; type.fields=program_.types[base].fields; type.fieldIds=program_.types[base].fieldIds; if(matchWord("and")){expectWord("implements",{"implements"});for(;;){const string interfaceName=identifier();const size_t interfaceId=resolveType(interfaceName);if(interfaceId==NO_ID||!program_.types[interfaceId].isInterface)error("unknown interface '"+interfaceName+"'");type.interfaces.push_back(interfaceId);if(!matchWord("and"))break;}} } else if (inheritanceKind == "implements") { for(;;){const string interfaceName=identifier();const size_t interfaceId=resolveType(interfaceName);if(interfaceId==NO_ID||!program_.types[interfaceId].isInterface)error("unknown interface '"+interfaceName+"'");type.interfaces.push_back(interfaceId);if(!matchWord("and"))break;} } else error("expected 'inherits' or 'implements' after 'that'"); } if(generic){expectWord("for",{"for"});expectWord("type",{"type"});(void)identifier();} if(!matchSymbol(":"))error("expected ':' after type declaration"); currentType_=program_.types.size();currentAccess_=isClass?AccessLevel::Private:AccessLevel::Public;program_.typeIds.emplace(name,currentType_);program_.types.push_back(move(type));return; } if (declarationKind != "function" && declarationKind != "method") error("expected function, method, class, structure, or interface after 'define'"); const bool method = declarationKind == "method"; if(method&&currentType_==NO_ID)error("methods must be defined inside a class or structure"); expectWord("named",{"named"});const string shortName=identifier(); if (program_.functions.size() >= MAX_FUNCTIONS) error("function-count limit exceeded"); FunctionInfo fn;fn.ownerType=method?currentType_:NO_ID;fn.generic=generic;fn.access=currentAccess_; fn.name=method?program_.types[currentType_].name+"::"+shortName:qualify(shortName); if(program_.functionIds.count(fn.name))error("function or method already defined: '"+fn.name+"'"); if(generic){expectWord("for",{"for"});expectWord("type",{"type"});fn.genericParameters.push_back(identifier());} if(matchWord("taking")){ auto appendParameter=[&](){ const string parameter=identifier(); if(find(fn.parameters.begin(),fn.parameters.end(),parameter)!=fn.parameters.end())error("duplicate parameter '"+parameter+"'"); if(fn.parameters.size()>=MAX_PARAMETERS)error("function parameter-count limit exceeded"); fn.parameters.push_back(parameter);StaticType type=StaticType::Unknown;string typeName;if(matchWord("as")){if(peek().kind!=TokenKind::Word)error("expected parameter type name");typeName=lowerAscii(peek().text);type=parseTypeAnnotation();}fn.parameterTypes.push_back(type);fn.parameterTypeNames.push_back(typeName);const size_t objectType=resolveType(typeName);if(objectType!=NO_ID)fn.localObjectTypes.emplace(parameter,objectType); }; if(matchSymbol("(")){ if(!matchSymbol(")"))for(;;){appendParameter();if(matchSymbol(")"))break;if(!matchSymbol(","))error("expected ',' or ')' in parameter list");} }else{ for(;;){appendParameter();if(matchWord("and"))continue;if(matchSymbol(","))continue;break;} } } if(!matchSymbol(":"))error("expected colon after function or method declaration"); const size_t id=program_.functions.size(); for(size_t parameter=0;parameter<fn.parameters.size();++parameter){const string &p=fn.parameters[parameter];fn.localIds.emplace(p,fn.localNames.size());fn.localNames.push_back(p);fn.localTypes.push_back(parameter<fn.parameterTypes.size()?fn.parameterTypes[parameter]:StaticType::Unknown);} program_.functionIds.emplace(fn.name,id);program_.functions.push_back(move(fn)); if(method)program_.types[currentType_].methods.emplace(shortName,id); currentAccess_=currentType_!=NO_ID?(program_.types[currentType_].isClass?AccessLevel::Private:AccessLevel::Public):AccessLevel::Public; if(!controls_.empty())error("function definitions cannot begin inside an unfinished control block"); activeFunction_=id; } void closeFunction(){ if(controls_.size())error("function ended while an if/while block remains open"); ++pos_;const bool method=matchWord("method");if(!method)expectWord("function",{"function"}); FunctionInfo&fn=program_.functions[activeFunction_]; if(fn.code.empty()||fn.code.back().op!=Op::Return){emit(Op::PushBoolean,0);emit(Op::Return);} if(fn.localNames.size()>MAX_LOCALS_PER_FUNCTION)error("function local-variable limit exceeded"); activeFunction_=0;optionalSemicolon(); } void closeType(){ if(!controls_.empty())error("type ended while a control block remains open"); ++pos_;const bool cls=matchWord("class");const bool structure=!cls&&matchWord("structure");const bool interfaceType=!cls&&!structure&&matchWord("interface"); if(!cls&&!structure&&!interfaceType)error("expected class, structure, or interface after end"); TypeInfo &type=program_.types.at(currentType_); if(!interfaceType){ for(size_t interfaceId:type.interfaces){ const TypeInfo &contract=program_.types.at(interfaceId); for(const auto &required:contract.requiredMethods){ const size_t methodId=methodFor(currentType_,required.first);if(methodId==NO_ID)error("type '"+type.name+"' does not implement required method '"+required.first+"'"); const FunctionInfo &fn=program_.functions.at(methodId);if(fn.parameters.size()!=required.second)error("method '"+required.first+"' has the wrong number of parameters for interface '"+contract.name+"'"); if(fn.access!=AccessLevel::Public)error("interface method implementation must be public: '"+required.first+"'"); } } } currentType_=NO_ID;currentAccess_=AccessLevel::Public;optionalSemicolon(); } Value parseFieldDefault(){ if(peek().kind==TokenKind::Number){Value v=peek().text.find('.')!=string::npos?Value(parseFiniteFloat(peek().text)):Value(BigNat(peek().text));++pos_;return v;} if(peek().kind==TokenKind::String){Value v=peek().text;++pos_;return v;} if(matchWord("true")) return true; if(matchWord("false")) return false; error("field defaults must be compile-time literals"); } void parseField(bool hasFieldKeyword = true){ if (hasFieldKeyword) ++pos_; if(program_.types[currentType_].isInterface)error("interfaces declare method requirements, not data fields"); const string name=identifier();TypeInfo&type=program_.types[currentType_];if(type.fieldIds.count(name))error("duplicate field '"+name+"'"); if (type.fields.size() >= MAX_FIELDS_PER_TYPE) error("field-count limit exceeded for type"); FieldInfo f;f.name=name;f.access=currentAccess_;f.declaringType=currentType_; if (!isWord("be") && peek().kind == TokenKind::Word && (peek(1).kind == TokenKind::Number || peek(1).kind == TokenKind::String || isWordAt(1, "true") || isWordAt(1, "false"))) { (void)suggestKeyword({"be"}, true, true); } if(matchWord("be"))f.initial=parseFieldDefault(); f.type=holds_alternative<BigNat>(f.initial)?StaticType::Natural:(holds_alternative<double>(f.initial)?StaticType::Floating:(holds_alternative<string>(f.initial)?StaticType::String:StaticType::Boolean)); type.fieldIds.emplace(name,type.fields.size());type.fields.push_back(move(f));expectSemicolon();currentAccess_=type.isClass?AccessLevel::Private:AccessLevel::Public; } void parseCompileTime(){ expectWord("during",{"during"});expectWord("compilation",{"compilation"}); string compileTimeAction; if (!matchOneOf({"generate", "define"}, compileTimeAction)) error("expected 'generate' or 'define' during compilation"); if (compileTimeAction == "generate") { if(peek().kind!=TokenKind::String)error("compile-time generate currently requires a quoted source fragment"); string generated=peek().text;++pos_; while (matchWord("plus") || (peek().kind == TokenKind::Word && suggestKeyword({"plus"}, true, true) && matchWord("plus"))) { if (peek().kind != TokenKind::String) error("compile-time string concatenation requires string literals"); generated += peek().text; ++pos_; if (generated.size() > MAX_GENERATED_SOURCE_BYTES) error("generated source exceeds 4 MiB limit"); } expectSemicolon();if(generated.size()>MAX_GENERATED_SOURCE_BYTES-generatedSourceBytes_)error("total generated source exceeds 4 MiB limit");generatedSourceBytes_+=generated.size(); vector<Token> generatedTokens=Lexer(generated).scan();if(!generatedTokens.empty()&&generatedTokens.back().kind==TokenKind::End)generatedTokens.pop_back(); if(generatedTokens.size()>MAX_TOKENS-tokens_.size())error("generated token count exceeds compiler limit"); tokens_.insert(tokens_.begin()+static_cast<ptrdiff_t>(pos_),make_move_iterator(generatedTokens.begin()),make_move_iterator(generatedTokens.end()));return; } expectWord("a",{"a"});expectWord("constant",{"constant"});expectWord("named",{"named"}); const string name=identifier();if(program_.constantIds.count(name))error("compile-time constant already defined: '"+name+"'");expectWord("be",{"be"}); Value value=parseFieldDefault();expectSemicolon();size_t id=program_.compileConstants.size();program_.compileConstants.push_back(move(value));program_.constantIds.emplace(name,id); } void parseVariableDeclaration(){ ++pos_;const string name=identifier();size_t id; if(activeFunction_==0)id=declareGlobal(name);else id=declareVariable(name); expectWord("be",{"be"});parseExpression();expectSemicolon(); if(activeFunction_==0)program_.globalTypes[id]=lastExpressionType_;else program_.functions[activeFunction_].localTypes[id]=lastExpressionType_; emit(activeFunction_==0?Op::StoreGlobal:Op::StoreLocal,id); } void parseSet(){ ++pos_;const Token targetToken=peek();bool isField=false;size_t id=0;bool global=false;string field;size_t memberType=NO_ID; if(isWord("this")){ ++pos_;field=identifier();memberType=currentType_;const FieldInfo *info=fieldFor(memberType,field);if(!info)error("unknown field '"+field+"'");if(!accessAllowed(info->access,info->declaringType))error("field '"+field+"' is not accessible here");emit(Op::LoadThis);isField=true; } else { const string name=identifier();VarRef r=resolveVariable(name);global=r.global;id=r.id;const size_t type=receiverType(name); if(matchWord("at")){ emitLoad(r);parseExpression({"to"});expectWord("to",{"to"});parseExpression();expectSemicolon();emit(Op::ArraySet);return; } if(peek().kind==TokenKind::Word&&type<program_.types.size()){ const string candidate=lowerAscii(peek().text);const FieldInfo *info=fieldFor(type,candidate); if(info){if(!accessAllowed(info->access,info->declaringType))error("field '"+candidate+"' is not accessible here");emitLoad(r);field=identifier();memberType=type;isField=true;} } } expectWord("to",{"to"});parseExpression();expectSemicolon(); if(isField){ const FieldInfo *info=fieldFor(memberType,field); if(info&&info->type!=StaticType::Unknown&&lastExpressionType_!=StaticType::Unknown&&info->type!=lastExpressionType_&&!(info->type==StaticType::Floating&&lastExpressionType_==StaticType::Natural))errorAt(targetToken,"assigned value has an incompatible inferred type for field '"+field+"'"); emit(Op::SetField,addString(field)); } else { FunctionInfo &function=program_.functions[activeFunction_];StaticType &oldType=global?program_.globalTypes.at(id):function.localTypes.at(id); if(oldType!=StaticType::Unknown&&lastExpressionType_!=StaticType::Unknown&&oldType!=lastExpressionType_&&!(oldType==StaticType::Floating&&lastExpressionType_==StaticType::Natural))errorAt(targetToken,"assigned value has an incompatible inferred type for variable '"+(global?program_.globals.at(id):function.localNames.at(id))+"'"); if(oldType==StaticType::Unknown)oldType=lastExpressionType_; emit(global?Op::StoreGlobal:Op::StoreLocal,id); } } void parseIncrease(bool decrease){ ++pos_;const string name=identifier();const VarRef r=resolveVariable(name);emitLoad(r);expectWord("by",{"by"});parseExpression();expectSemicolon();emit(decrease?Op::Subtract:Op::Add);emitStore(r); } void parseDisplay(){++pos_;parseExpression();expectSemicolon();emit(Op::Print);} void parseRead(){++pos_;const string name=identifier();const VarRef r=resolveVariable(name);expectSemicolon();emit(r.global?Op::Read:Op::Read,r.id,r.global?1:0);} void parseIf(){ ++pos_;parseExpression({"then"});(void)matchWord("then");if(!matchSymbol(":"))error("expected ':' after the if condition"); const size_t jump=program_.functions[activeFunction_].code.size();emit(Op::JumpIfFalse,0);controls_.push_back({Control::If,jump,0,NO_ID,false}); } void parseWhile(){ ++pos_;const size_t start=program_.functions[activeFunction_].code.size();parseExpression();if(!matchSymbol(":"))error("expected ':' after the while condition"); const size_t jump=program_.functions[activeFunction_].code.size();emit(Op::JumpIfFalse,0);controls_.push_back({Control::While,jump,start,NO_ID,false}); } void parseTry(){ ++pos_;if(!matchSymbol(":"))error("expected ':' after try"); const size_t handler=program_.functions[activeFunction_].code.size();emit(Op::PushHandler,0,0);controls_.push_back({Control::Try,handler,0,NO_ID,false}); } void handleCatch(){ ++pos_;if(controls_.empty()||controls_.back().kind!=Control::Try||controls_.back().hasElse)error("catch without a matching try"); const string errorName=identifier();const size_t local=declareVariable(errorName);Control &control=controls_.back(); emit(Op::PopHandler);const size_t skip=program_.functions[activeFunction_].code.size();emit(Op::Jump,0); auto &handler=program_.functions[activeFunction_].code.at(control.branch);handler.a=program_.functions[activeFunction_].code.size();handler.b=local; control.endJump=skip;control.hasElse=true;if(!matchSymbol(":"))error("expected ':' after catch binding"); } void handleFinally(){ ++pos_; if(controls_.empty()||controls_.back().kind!=Control::Try||!controls_.back().hasElse||controls_.back().hasFinally) error("finally requires one preceding catch clause and a matching try block"); Control &control=controls_.back(); patch(control.endJump,program_.functions[activeFunction_].code.size()); control.hasFinally=true; if(!matchSymbol(":"))error("expected ':' after finally"); } void handleElse(){ ++pos_;if(controls_.empty()||controls_.back().kind!=Control::If||controls_.back().hasElse)error("'else' without a matching 'if'"); Control&c=controls_.back();const size_t endJump=program_.functions[activeFunction_].code.size();emit(Op::Jump,0);patch(c.branch,program_.functions[activeFunction_].code.size());c.endJump=endJump;c.hasElse=true; if (!matchSymbol(":")) optionalSemicolon(); } void closeControl(){ ++pos_;const bool isIf=matchWord("if");const bool isWhile=!isIf&&matchWord("while");const bool isTry=!isIf&&!isWhile&&matchWord("try"); if(controls_.empty())error("'end' without an open block"); Control c=controls_.back();controls_.pop_back(); if((isIf&&c.kind!=Control::If)||(isWhile&&c.kind!=Control::While)||(isTry&&c.kind!=Control::Try)||(!isIf&&!isWhile&&!isTry))error("block terminator does not match the open block"); if(c.kind==Control::While){emit(Op::Jump,c.loopStart);patch(c.branch,program_.functions[activeFunction_].code.size());} else if(c.kind==Control::If){if(c.hasElse)patch(c.endJump,program_.functions[activeFunction_].code.size());else patch(c.branch,program_.functions[activeFunction_].code.size());} else {if(!c.hasElse)error("try block requires a catch block");if(!c.hasFinally)patch(c.endJump,program_.functions[activeFunction_].code.size());} optionalSemicolon(); } void parseObjectCreation(){ expectWord("create",{"create"});expectWord("an",{"an"});expectWord("object",{"object"});expectWord("of",{"of"});string typeName=identifier();while(peek().kind==TokenKind::Word&&!isWord("named"))typeName+="::"+identifier();const size_t typeId=resolveType(typeName);if(typeId==NO_ID)error("unknown type '"+typeName+"'");if(program_.types[typeId].isInterface)error("interfaces cannot be instantiated");expectWord("named",{"named"});const string var=identifier();size_t id=activeFunction_==0?declareGlobal(var):declareVariable(var);if(activeFunction_==0)program_.globalObjectTypes[var]=typeId;else program_.functions[activeFunction_].localObjectTypes[var]=typeId;expectSemicolon();emit(Op::NewObject,typeId);emit(activeFunction_==0?Op::StoreGlobal:Op::StoreLocal,id); } void parseCallStatement(){ ++pos_; const size_t expressionStart=pos_; if(peek().kind!=TokenKind::Word)error("expected function or object name after call"); const string first=identifier(); const size_t receiver=receiverType(first); if(peek().kind==TokenKind::Word&&receiver<program_.types.size()&&isSymbolAt(pos_+1,"(")){ const string method=lowerAscii(peek().text);const size_t methodId=methodFor(receiver,method); if(methodId!=NO_ID&&!accessAllowed(program_.functions[methodId].access,program_.functions[methodId].ownerType))error("method '"+method+"' is not accessible here"); } pos_=expressionStart; parseExpression(); expectSemicolon(); emit(Op::Pop); } void parseStatement(){ static const vector<string> statements={"let","set","increase","decrease","display","read","if","while","try","create","call","append","throw","give","return"}; for(int attempt=0;attempt<2;++attempt){ if(isWord("let")){parseVariableDeclaration();return;}if(isWord("set")){parseSet();return;} if(isWord("increase")){parseIncrease(false);return;}if(isWord("decrease")){parseIncrease(true);return;} if(isWord("display")){parseDisplay();return;}if(isWord("read")){parseRead();return;} if(isWord("if")){parseIf();return;}if(isWord("while")){parseWhile();return;} if(isWord("try")){parseTry();return;} if(isWord("create")){parseObjectCreation();return;}if(isWord("call")){parseCallStatement();return;} if(isWord("append")){++pos_;parseExpression({"to"});expectWord("to",{"to"});const string arrayName=identifier();const VarRef array=resolveVariable(arrayName);emitLoad(array);expectSemicolon();emit(Op::ArrayAppend);return;} if(isWord("throw")){++pos_;parseExpression();expectSemicolon();emit(Op::Throw);return;} if(isWord("give")||isWord("return")){ if(matchWord("give"))expectWord("back",{"back"});else ++pos_; parseExpression();expectSemicolon(); FunctionInfo &function=program_.functions[activeFunction_]; if(function.returnType==StaticType::Unknown)function.returnType=lastExpressionType_; else if(lastExpressionType_!=StaticType::Unknown&&lastExpressionType_!=function.returnType&&!(lastExpressionType_==StaticType::Natural&&function.returnType==StaticType::Floating))error("inconsistent statically inferred return type"); emit(Op::Return);return; } if(activeFunction_==0&&currentType_!=NO_ID)error("inside a class/structure, expected a field, method definition, or end marker"); if(attempt==0&&suggestKeyword(statements, true, true))continue; break; } error("unknown statement; expected a statement keyword"); } }; // --------------------------- Iterative virtual machine ------------------- class VirtualMachine { struct RuntimeObject { size_t type=0; vector<Value> fields; StringMap<Value> mapValues; size_t payloadBytes=0; bool marked=false; bool isArray=false; bool isMap=false; }; struct Handler { size_t target=0, local=0, stackDepth=0; }; struct Frame { size_t function=0,pc=0,stackBase=0;vector<Value> locals;optional<uint32_t> receiver;vector<Handler> handlers; }; public: VirtualMachine(const Program&p,uint64_t gas):p_(p),gasLimit_(gas),globals_(p.globals.size(),BigNat(0)){} void run(){ if(p_.functions.empty())throw runtime_error("program has no entry function"); vector<Value> stack;stack.reserve(128);vector<Frame> frames;frames.reserve(32); frames.push_back(makeFrame(0,{},nullopt,0));uint64_t gas=0; auto pop=[&]()->Value{if(stack.empty())throw runtime_error("runtime stack underflow");Value v=move(stack.back());stack.pop_back();return v;}; auto push=[&](Value v){if(stack.size()>=MAX_RUNTIME_STACK)throw runtime_error("runtime operand stack limit exceeded");stack.push_back(move(v));}; while(!frames.empty()){ Frame&f=frames.back();const FunctionInfo&fn=p_.functions.at(f.function); if(f.pc>=fn.code.size()){ const size_t base=f.stackBase;const bool isMain=frames.size()==1;frames.pop_back();stack.resize(base);if(!isMain)push(false);continue; } const Instruction ins=fn.code[f.pc++]; uint64_t cost = 1; if (ins.op == Op::Multiply && stack.size() >= 2 && holds_alternative<BigNat>(stack.back()) && holds_alternative<BigNat>(stack[stack.size()-2])) { const uint64_t a = get<BigNat>(stack.back()).limbCount(); const uint64_t b = get<BigNat>(stack[stack.size()-2]).limbCount(); cost += a * b; } else if ((ins.op == Op::Add || ins.op == Op::Subtract) && stack.size() >= 2 && holds_alternative<BigNat>(stack.back()) && holds_alternative<BigNat>(stack[stack.size()-2])) { cost += max(get<BigNat>(stack.back()).limbCount(), get<BigNat>(stack[stack.size()-2]).limbCount()); } else if (ins.op == Op::Add && stack.size() >= 2 && holds_alternative<string>(stack.back()) && holds_alternative<string>(stack[stack.size()-2])) { cost += (get<string>(stack.back()).size() + get<string>(stack[stack.size()-2]).size()) / 64; } else if (ins.op == Op::Print && !stack.empty()) { if (holds_alternative<string>(stack.back())) cost += get<string>(stack.back()).size() / 64; else if (holds_alternative<BigNat>(stack.back())) cost += get<BigNat>(stack.back()).limbCount(); } else if (ins.op == Op::NewObject && ins.a < p_.types.size()) { cost += p_.types[ins.a].fields.size(); } else if (ins.op == Op::NewArray || ins.op == Op::NewMap) { cost += ins.b; } else if (ins.op == Op::ArrayAppend && !stack.empty()) { if (holds_alternative<string>(stack.back())) cost += get<string>(stack.back()).size() / 64; else if (holds_alternative<BigNat>(stack.back())) cost += get<BigNat>(stack.back()).limbCount(); } if (cost > numeric_limits<uint64_t>::max() - gas) throw runtime_error("execution gas counter overflow"); gas += cost; if(gasLimit_!=0 && gas>gasLimit_)throw runtime_error("execution gas limit exceeded; possible infinite loop or excessive operation cost"); try { switch(ins.op){ case Op::PushNumber: push(p_.compileConstants.at(ins.a));break; case Op::PushString: push(p_.strings.at(ins.a));break; case Op::PushBoolean: push(ins.a!=0);break; case Op::LoadLocal: push(f.locals.at(ins.a));break; case Op::StoreLocal: f.locals.at(ins.a)=pop();break; case Op::LoadGlobal: push(globals_.at(ins.a));break; case Op::StoreGlobal: globals_.at(ins.a)=pop();break; case Op::LoadThis: if(!f.receiver)throw runtime_error("'this' used outside a method");push(ObjectRef{*f.receiver});break; case Op::NewObject:{ const TypeInfo& t = p_.types.at(ins.a); size_t payload = 0; for (const FieldInfo& field : t.fields) payload = checkedPayloadAdd(payload, slotPayloadBytes(field.initial)); if (allocationsSinceGc_ >= nextGcThreshold_ || liveObjects_ >= MAX_OBJECTS || t.fields.size() > MAX_RUNTIME_OBJECT_FIELDS - min(liveFieldSlots_, MAX_RUNTIME_OBJECT_FIELDS)) collectGarbage(stack, frames); ensureHeapCapacity(payload, stack, frames); if (liveObjects_ >= MAX_OBJECTS || t.fields.size() > MAX_RUNTIME_OBJECT_FIELDS - min(liveFieldSlots_, MAX_RUNTIME_OBJECT_FIELDS)) throw runtime_error("runtime object memory limit exceeded"); RuntimeObject obj; obj.type = ins.a; obj.payloadBytes = payload; obj.fields.reserve(t.fields.size()); for (const FieldInfo& field : t.fields) obj.fields.push_back(field.initial); uint32_t id = 0; if (!freeObjectIds_.empty()) { id = freeObjectIds_.back(); freeObjectIds_.pop_back(); objects_.at(id) = move(obj); } else { if (objects_.size() >= MAX_OBJECTS) throw runtime_error("live object limit exceeded after garbage collection"); id = static_cast<uint32_t>(objects_.size()); objects_.emplace_back(move(obj)); } ++liveObjects_; liveFieldSlots_ += t.fields.size(); liveHeapPayloadBytes_ += payload; ++allocationsSinceGc_; push(ObjectRef{id}); break; } case Op::NewArray:{ if(ins.b>MAX_ARRAY_ELEMENTS||ins.b>stack.size())throw runtime_error("invalid array literal size"); size_t payload = 0; for (size_t i = stack.size() - ins.b; i < stack.size(); ++i) payload = checkedPayloadAdd(payload, slotPayloadBytes(stack[i])); if(allocationsSinceGc_>=nextGcThreshold_||liveObjects_>=MAX_OBJECTS||ins.b>MAX_RUNTIME_OBJECT_FIELDS-min(liveFieldSlots_,MAX_RUNTIME_OBJECT_FIELDS))collectGarbage(stack,frames); ensureHeapCapacity(payload, stack, frames); if(liveObjects_>=MAX_OBJECTS||ins.b>MAX_RUNTIME_OBJECT_FIELDS-min(liveFieldSlots_,MAX_RUNTIME_OBJECT_FIELDS))throw runtime_error("array allocation limit exceeded"); vector<Value> values(ins.b); for(size_t i=ins.b;i>0;--i){values[i-1]=move(stack.back());stack.pop_back();} RuntimeObject object;object.isArray=true;object.fields=move(values);object.payloadBytes=payload;uint32_t id=0; if(!freeObjectIds_.empty()){id=freeObjectIds_.back();freeObjectIds_.pop_back();objects_.at(id)=move(object);}else{id=static_cast<uint32_t>(objects_.size());objects_.emplace_back(move(object));} ++liveObjects_;liveFieldSlots_+=objects_[id]->fields.size();liveHeapPayloadBytes_+=payload;++allocationsSinceGc_;push(ObjectRef{id});break; } case Op::NewMap:{ if(ins.b%2!=0||ins.b>MAX_ARRAY_ELEMENTS*2||ins.b>stack.size())throw runtime_error("invalid map constructor size"); size_t payload = 0; for (size_t i = stack.size() - ins.b; i < stack.size(); i += 2) { if (!holds_alternative<string>(stack[i])) throw runtime_error("map constructor keys must be strings"); const auto& key = get<string>(stack[i]); if (key.size() > MAX_RUNTIME_HEAP_BYTES) throw runtime_error("map key exceeds runtime heap limit"); payload = checkedPayloadAdd(payload, sizeof(Value) + RUNTIME_SLOT_OVERHEAD + key.size()); payload = checkedPayloadAdd(payload, valuePayloadBytes(stack[i + 1])); } if(allocationsSinceGc_>=nextGcThreshold_||liveObjects_>=MAX_OBJECTS||ins.b/2>MAX_RUNTIME_OBJECT_FIELDS-min(liveFieldSlots_,MAX_RUNTIME_OBJECT_FIELDS))collectGarbage(stack,frames); ensureHeapCapacity(payload, stack, frames); if(liveObjects_>=MAX_OBJECTS||ins.b/2>MAX_RUNTIME_OBJECT_FIELDS-min(liveFieldSlots_,MAX_RUNTIME_OBJECT_FIELDS))throw runtime_error("map allocation limit exceeded"); vector<Value> values(ins.b); for(size_t i=ins.b;i>0;--i){values[i-1]=move(stack.back());stack.pop_back();} RuntimeObject object;object.isMap=true;object.payloadBytes=payload;object.mapValues.reserve(ins.b/2); for(size_t i=0;i<values.size();i+=2){ if(!holds_alternative<string>(values[i]))throw runtime_error("map constructor keys must be strings"); if(!object.mapValues.emplace(get<string>(values[i]),move(values[i+1])).second)throw runtime_error("duplicate map key"); } uint32_t id=0; if(!freeObjectIds_.empty()){id=freeObjectIds_.back();freeObjectIds_.pop_back();objects_.at(id)=move(object);} else {if(objects_.size()>=MAX_OBJECTS)throw runtime_error("object slot limit exceeded");id=static_cast<uint32_t>(objects_.size());objects_.emplace_back(move(object));} ++liveObjects_;liveFieldSlots_+=ins.b/2;liveHeapPayloadBytes_+=payload;++allocationsSinceGc_;push(ObjectRef{id});break; } case Op::ArrayGet:{ Value indexValue=pop(),arrayValue=pop();RuntimeObject&array=objectFrom(arrayValue); if(array.isMap){if(!holds_alternative<string>(indexValue))throw runtime_error("map lookup key must be a string");const string&key=get<string>(indexValue);auto found=array.mapValues.find(key);if(found==array.mapValues.end())throw runtime_error("map key was not found: "+key);push(found->second);} else {if(!array.isArray)throw runtime_error("'at' requires an array or map collection");const size_t index=arrayIndex(indexValue);if(index>=array.fields.size())throw runtime_error("array index is out of range");push(array.fields[index]);}break; } case Op::ArraySet:{ Value value=pop(),indexValue=pop(),arrayValue=pop();RuntimeObject&array=objectFrom(arrayValue); if(array.isMap){ if(!holds_alternative<string>(indexValue))throw runtime_error("map assignment key must be a string"); const string&key=get<string>(indexValue);auto found=array.mapValues.find(key); if(found==array.mapValues.end()){ if(liveFieldSlots_>=MAX_RUNTIME_OBJECT_FIELDS)throw runtime_error("map element limit exceeded"); const size_t added=checkedPayloadAdd(sizeof(Value)+RUNTIME_SLOT_OVERHEAD+key.size(),valuePayloadBytes(value)); if(added>MAX_RUNTIME_HEAP_BYTES-liveHeapPayloadBytes_)throw runtime_error("runtime heap payload limit exceeded"); array.mapValues.emplace(key,move(value));++liveFieldSlots_;array.payloadBytes+=added;liveHeapPayloadBytes_+=added; }else{ const size_t oldBytes=valuePayloadBytes(found->second),newBytes=valuePayloadBytes(value); replaceHeapPayload(array,oldBytes,newBytes);found->second=move(value); } } else { if(!array.isArray)throw runtime_error("indexed assignment requires an array or map"); const size_t index=arrayIndex(indexValue);if(index>=array.fields.size())throw runtime_error("array index is out of range"); const size_t oldBytes=valuePayloadBytes(array.fields[index]),newBytes=valuePayloadBytes(value); replaceHeapPayload(array,oldBytes,newBytes);array.fields[index]=move(value); }break; } case Op::ArrayLength:{ Value arrayValue=pop();RuntimeObject&array=objectFrom(arrayValue);if(!array.isArray&&!array.isMap)throw runtime_error("length requires an array or map collection");push(BigNat(static_cast<uint64_t>(array.isMap?array.mapValues.size():array.fields.size())));break; } case Op::ArrayAppend:{ Value arrayValue=pop(),value=pop();RuntimeObject&array=objectFrom(arrayValue); if(!array.isArray)throw runtime_error("append requires an array"); if(array.fields.size()>=MAX_ARRAY_ELEMENTS||liveFieldSlots_>=MAX_RUNTIME_OBJECT_FIELDS)throw runtime_error("array element limit exceeded"); const size_t added=slotPayloadBytes(value); if(added>MAX_RUNTIME_HEAP_BYTES-liveHeapPayloadBytes_)throw runtime_error("runtime heap payload limit exceeded"); array.fields.push_back(move(value));array.payloadBytes+=added;liveHeapPayloadBytes_+=added;++liveFieldSlots_;break; } case Op::GetField:{ Value base=pop();RuntimeObject&o=objectFrom(base);if(o.isArray||o.isMap)throw runtime_error("collection values do not have named object fields");const string&name=p_.strings.at(ins.a);auto it=p_.types.at(o.type).fieldIds.find(name);if(it==p_.types.at(o.type).fieldIds.end())throw runtime_error("object has no field '"+name+"'");push(o.fields.at(it->second));break; } case Op::SetField:{ Value value=pop();Value base=pop();RuntimeObject&o=objectFrom(base);if(o.isArray||o.isMap)throw runtime_error("collection values do not have named object fields");const string&name=p_.strings.at(ins.a);auto it=p_.types.at(o.type).fieldIds.find(name);if(it==p_.types.at(o.type).fieldIds.end())throw runtime_error("object has no field '"+name+"'");Value&slot=o.fields.at(it->second);replaceHeapPayload(o,valuePayloadBytes(slot),valuePayloadBytes(value));slot=move(value);break; } case Op::Add:{Value b=pop(),a=pop();if(holds_alternative<BigNat>(a)&&holds_alternative<BigNat>(b))push(add(get<BigNat>(a),get<BigNat>(b)));else if(isNumeric(a)&&isNumeric(b))push(checkedFloat(numericAsDouble(a)+numericAsDouble(b)));else if(holds_alternative<string>(a)&&holds_alternative<string>(b)){string text=get<string>(a);const auto&right=get<string>(b);if(right.size()>1024u*1024u-text.size())throw runtime_error("string concatenation limit exceeded");text+=right;push(move(text));}else throw runtime_error("plus requires two numbers or two strings");break;} case Op::Subtract:{Value b=pop(),a=pop();if(holds_alternative<BigNat>(a)&&holds_alternative<BigNat>(b))push(subtract(get<BigNat>(a),get<BigNat>(b)));else if(isNumeric(a)&&isNumeric(b))push(checkedFloat(numericAsDouble(a)-numericAsDouble(b)));else throw runtime_error("minus requires numbers");break;} case Op::Multiply:{Value b=pop(),a=pop();if(holds_alternative<BigNat>(a)&&holds_alternative<BigNat>(b))push(multiply(get<BigNat>(a),get<BigNat>(b)));else if(isNumeric(a)&&isNumeric(b))push(checkedFloat(numericAsDouble(a)*numericAsDouble(b)));else throw runtime_error("times requires numbers");break;} case Op::Divide:{Value b=pop(),a=pop();if(!isNumeric(a)||!isNumeric(b))throw runtime_error("division requires numbers");const double divisor=numericAsDouble(b);if(divisor==0.0)throw runtime_error("division by zero");push(checkedFloat(numericAsDouble(a)/divisor));break;} case Op::Remainder:{Value b=pop(),a=pop();if(!isNumeric(a)||!isNumeric(b))throw runtime_error("modulo requires numbers");const double divisor=numericAsDouble(b);if(divisor==0.0)throw runtime_error("modulo by zero");push(checkedFloat(fmod(numericAsDouble(a),divisor)));break;} case Op::Equal:case Op::NotEqual:case Op::Less:case Op::LessEqual:case Op::Greater:case Op::GreaterEqual:{Value b=pop(),a=pop();int c=compareValues(a,b);bool v=false; if(ins.op==Op::Equal)v=c==0;else if(ins.op==Op::NotEqual)v=c!=0;else if(ins.op==Op::Less)v=c<0;else if(ins.op==Op::LessEqual)v=c<=0;else if(ins.op==Op::Greater)v=c>0;else v=c>=0;push(v);break;} case Op::BooleanAnd:{Value b=pop(),a=pop();push(truthy(a)&&truthy(b));break;} case Op::BooleanOr:{Value b=pop(),a=pop();push(truthy(a)||truthy(b));break;} case Op::BooleanNot:{Value a=pop();push(!truthy(a));break;} case Op::Jump:if(ins.a>fn.code.size())throw runtime_error("invalid jump target");f.pc=ins.a;break; case Op::JumpIfFalse:{Value c=pop();if(!truthy(c)){if(ins.a>fn.code.size())throw runtime_error("invalid conditional jump target");f.pc=ins.a;}break;} case Op::Call:{ const string&name=p_.strings.at(ins.a);auto it=p_.functionIds.find(name);if(it==p_.functionIds.end())throw runtime_error("undefined function '"+name+"'"); vector<Value>args=takeArguments(stack,ins.b);const size_t base=stack.size();if(frames.size()>=MAX_CALL_FRAMES)throw runtime_error("maximum call-frame depth exceeded");frames.push_back(makeFrame(it->second,move(args),nullopt,base));break; } case Op::CallMethod:{ vector<Value>args=takeArguments(stack,ins.b);Value recv=pop();RuntimeObject&o=objectFrom(recv);const string&name=p_.strings.at(ins.a);size_t type=o.type,method=numeric_limits<size_t>::max(); for(size_t hops=0;type<p_.types.size()&&hops<=p_.types.size();++hops){auto it=p_.types[type].methods.find(name);if(it!=p_.types[type].methods.end()){method=it->second;break;}if(p_.types[type].base==numeric_limits<size_t>::max())break;type=p_.types[type].base;} if(method==numeric_limits<size_t>::max())throw runtime_error("method '"+name+"' is not defined for this object"); const size_t base=stack.size();if(frames.size()>=MAX_CALL_FRAMES)throw runtime_error("maximum call-frame depth exceeded");frames.push_back(makeFrame(method,move(args),get<ObjectRef>(recv).id,base));break; } case Op::Return:{Value result=stack.empty()?Value(false):pop();const size_t base=f.stackBase;const bool isMain=frames.size()==1;frames.pop_back();stack.resize(base);if(!isMain)push(move(result));break;} case Op::Print:{Value v=pop();printValue(v);break;} case Op::Read:{ string input; int ch = 0; do { ch = cin.get(); } while (ch != char_traits<char>::eof() && isspace(static_cast<unsigned char>(ch))); if (ch == char_traits<char>::eof()) throw runtime_error("input operation failed: end of input"); input.reserve(64); do { if (input.size() >= MAX_INPUT_TOKEN_BYTES) throw runtime_error("input token exceeds 100000-byte safety limit"); input.push_back(static_cast<char>(ch)); ch = cin.get(); } while (ch != char_traits<char>::eof() && !isspace(static_cast<unsigned char>(ch))); Value v; if (!input.empty() && input.size() <= MAX_INTEGER_DIGITS && all_of(input.begin(),input.end(),[](char c){return isAsciiDigit(u8(c));})) v=BigNat(input); else if(input.find('.')!=string::npos) v=parseFiniteFloat(input); else v=move(input); if(ins.b==1)globals_.at(ins.a)=move(v);else f.locals.at(ins.a)=move(v); break; } case Op::PushHandler:{if(ins.a>fn.code.size()||ins.b>=f.locals.size())throw runtime_error("invalid exception-handler metadata");f.handlers.push_back({ins.a,ins.b,stack.size()});break;} case Op::PopHandler:if(f.handlers.empty())throw runtime_error("exception-handler stack underflow");else f.handlers.pop_back();break; case Op::Throw:{Value value=pop();throw runtime_error(valueToString(value));} case Op::Pop:(void)pop();break; } } catch(const exception &runtimeError) { bool handled=false; for(size_t frameIndex=frames.size();frameIndex>0;--frameIndex){ Frame &targetFrame=frames[frameIndex-1]; if(!targetFrame.handlers.empty()){ const Handler handler=targetFrame.handlers.back();targetFrame.handlers.pop_back(); frames.resize(frameIndex);stack.resize(handler.stackDepth); if(handler.local>=targetFrame.locals.size()||handler.target>p_.functions[targetFrame.function].code.size())throw runtime_error("corrupt exception-handler state"); const string message=runtimeError.what();targetFrame.locals[handler.local]=message.substr(0,MAX_INPUT_TOKEN_BYTES); targetFrame.pc=handler.target;handled=true;break; } } if(!handled)throw; } } } private: const Program&p_;uint64_t gasLimit_;vector<Value>globals_; vector<optional<RuntimeObject>> objects_; vector<uint32_t> freeObjectIds_; size_t liveObjects_=0, liveFieldSlots_=0, liveHeapPayloadBytes_=0, allocationsSinceGc_=0, nextGcThreshold_=1024; uint64_t outputBytes_=0; static size_t valuePayloadBytes(const Value& value) noexcept { if (const auto* text = get_if<string>(&value)) return text->size(); if (const auto* natural = get_if<BigNat>(&value)) return natural->limbCount() * sizeof(uint32_t); return 0; } static size_t checkedPayloadAdd(size_t total, size_t amount) { if (amount > MAX_RUNTIME_HEAP_BYTES || total > MAX_RUNTIME_HEAP_BYTES - amount) throw runtime_error("runtime heap payload exceeds 128 MiB safety limit"); return total + amount; } static size_t slotPayloadBytes(const Value& value) { return checkedPayloadAdd(sizeof(Value) + RUNTIME_SLOT_OVERHEAD, valuePayloadBytes(value)); } void ensureHeapCapacity(size_t requested, const vector<Value>& stack, const vector<Frame>& frames) { if (requested > MAX_RUNTIME_HEAP_BYTES) throw runtime_error("single allocation exceeds 128 MiB runtime heap limit"); if (requested > MAX_RUNTIME_HEAP_BYTES - min(liveHeapPayloadBytes_, MAX_RUNTIME_HEAP_BYTES)) collectGarbage(stack, frames); if (requested > MAX_RUNTIME_HEAP_BYTES - min(liveHeapPayloadBytes_, MAX_RUNTIME_HEAP_BYTES)) throw runtime_error("runtime heap payload limit exceeded"); } void replaceHeapPayload(RuntimeObject& object, size_t oldBytes, size_t newBytes) { if (oldBytes > object.payloadBytes || oldBytes > liveHeapPayloadBytes_) throw runtime_error("runtime heap accounting invariant failed"); const size_t base = liveHeapPayloadBytes_ - oldBytes; if (newBytes > MAX_RUNTIME_HEAP_BYTES - min(base, MAX_RUNTIME_HEAP_BYTES)) throw runtime_error("runtime heap payload limit exceeded"); object.payloadBytes = object.payloadBytes - oldBytes + newBytes; liveHeapPayloadBytes_ = base + newBytes; } static bool isNumeric(const Value &value) { return holds_alternative<BigNat>(value) || holds_alternative<double>(value); } static double numericAsDouble(const Value &value) { if(const auto *floating=get_if<double>(&value))return *floating; const string digits=get<BigNat>(value).toString(); char *end=nullptr;const double result=strtod(digits.c_str(),&end); if(!end||*end!='\0'||!isfinite(result))throw runtime_error("integer is too large for floating-point conversion"); return result; } static double checkedFloat(double value) { if(!isfinite(value)||fabs(value)>1.0e300)throw runtime_error("floating-point result is outside the supported finite range"); return value; } static string valueToString(const Value &value) { if(const auto *natural=get_if<BigNat>(&value))return natural->toString(); if(const auto *floating=get_if<double>(&value)){ostringstream out;out<<setprecision(17)<<*floating;return out.str();} if(const auto *boolean=get_if<bool>(&value))return *boolean?"true":"false"; if(const auto *text=get_if<string>(&value))return *text; return "[object #"+to_string(get<ObjectRef>(value).id)+"]"; } static size_t arrayIndex(const Value &value) { const auto *natural=get_if<BigNat>(&value);if(!natural)throw runtime_error("array index must be a non-negative integer"); const string digits=natural->toString();if(digits.size()>18)throw runtime_error("array index is too large"); return static_cast<size_t>(stoull(digits)); } void markValue(const Value& value, vector<uint32_t>& work) { const auto* ref = get_if<ObjectRef>(&value); if (!ref || ref->id >= objects_.size() || !objects_[ref->id]) return; RuntimeObject& object = *objects_[ref->id]; if (!object.marked) { object.marked = true; work.push_back(ref->id); } } void collectGarbage(const vector<Value>& stack, const vector<Frame>& frames) { vector<uint32_t> work; work.reserve(liveObjects_); for (const Value& value : globals_) markValue(value, work); for (const Value& value : stack) markValue(value, work); for (const Frame& frame : frames) { for (const Value& value : frame.locals) markValue(value, work); if (frame.receiver) markValue(Value(ObjectRef{*frame.receiver}), work); } while (!work.empty()) { const uint32_t id = work.back(); work.pop_back(); RuntimeObject& object = *objects_[id]; for (const Value& field : object.fields) markValue(field, work); for (const auto &entry : object.mapValues) markValue(entry.second, work); } for (size_t i = 0; i < objects_.size(); ++i) { if (!objects_[i]) continue; if (!objects_[i]->marked) { liveFieldSlots_ -= objects_[i]->fields.size()+objects_[i]->mapValues.size(); if (objects_[i]->payloadBytes > liveHeapPayloadBytes_) throw runtime_error("runtime heap accounting invariant failed during collection"); liveHeapPayloadBytes_ -= objects_[i]->payloadBytes; objects_[i].reset(); freeObjectIds_.push_back(static_cast<uint32_t>(i)); --liveObjects_; } else { objects_[i]->marked = false; } } allocationsSinceGc_ = 0; nextGcThreshold_ = max<size_t>(1024, liveObjects_ * 2); } Frame makeFrame(size_t id,vector<Value>args,optional<uint32_t>receiver,size_t base){ const FunctionInfo&fn=p_.functions.at(id);if(args.size()!=fn.parameters.size())throw runtime_error("function '"+fn.name+"' expects "+to_string(fn.parameters.size())+" arguments but received "+to_string(args.size())); Frame f;f.function=id;f.stackBase=base;f.receiver=receiver;f.locals.assign(fn.localNames.size(),BigNat(0));for(size_t i=0;i<args.size();++i)f.locals.at(i)=move(args[i]);return f; } static vector<Value> takeArguments(vector<Value>&stack,size_t n){ if(n>stack.size()) throw runtime_error("not enough values on the runtime stack for function arguments"); vector<Value>args(n); for(size_t i=n;i>0;--i){args[i-1]=move(stack.back());stack.pop_back();} return args; } RuntimeObject&objectFrom(const Value&v){if(!holds_alternative<ObjectRef>(v))throw runtime_error("property or method access requires an object");const size_t id=get<ObjectRef>(v).id;if(id>=objects_.size()||!objects_[id])throw runtime_error("invalid or collected object reference");return *objects_[id];} static bool truthy(const Value&v){if(holds_alternative<BigNat>(v))return !get<BigNat>(v).isZero();if(holds_alternative<double>(v))return get<double>(v)!=0.0;if(holds_alternative<bool>(v))return get<bool>(v);if(holds_alternative<string>(v))return !get<string>(v).empty();return true;} static int compareValues(const Value&a,const Value&b){ if(isNumeric(a)&&isNumeric(b)){const double x=numericAsDouble(a),y=numericAsDouble(b);return x<y?-1:(x>y?1:0);} if(a.index()!=b.index())throw runtime_error("comparison requires compatible value types"); if(holds_alternative<BigNat>(a))return compare(get<BigNat>(a),get<BigNat>(b)); if(holds_alternative<bool>(a)){bool x=get<bool>(a),y=get<bool>(b);return x==y?0:(x?1:-1);} if(holds_alternative<string>(a)){const auto&x=get<string>(a);const auto&y=get<string>(b);return x<y?-1:(x>y?1:0);} const auto x=get<ObjectRef>(a).id,y=get<ObjectRef>(b).id;return x==y?0:(x<y?-1:1); } void printValue(const Value&v){ string text; if(holds_alternative<BigNat>(v)) text=get<BigNat>(v).toString(); else if(holds_alternative<double>(v)){ostringstream out;out<<setprecision(17)<<get<double>(v);text=out.str();} else if(holds_alternative<bool>(v)) text=get<bool>(v)?"true":"false"; else if(holds_alternative<string>(v)) text=get<string>(v); else text="[object #"+to_string(get<ObjectRef>(v).id)+"]"; const uint64_t bytes=static_cast<uint64_t>(text.size())+1; if(bytes>MAX_OUTPUT_BYTES-outputBytes_) throw runtime_error("program output exceeds 64 MiB safety limit"); cout.write(text.data(),static_cast<streamsize>(text.size())); cout.put('\n'); if(!cout) throw runtime_error("output stream failure"); outputBytes_+=bytes; } }; // ----------------------- Sandboxed source/include loading ---------------- // This path layer intentionally avoids std::filesystem because MinGW-w64 // GCC 8.1 ships a broken <filesystem> implementation. Canonicalization uses // OS APIs so symlinks/reparse points are resolved before include-root checks. static string normalizePathSeparators(string path) { for (char &c : path) if (c == '\\') c = '/'; return path; } static bool isAbsolutePathString(const string &path) noexcept { if (path.empty()) return false; #ifdef _WIN32 return path[0] == '/' || path[0] == '\\' || (path.size() >= 2 && isAsciiAlpha(u8(path[0])) && path[1] == ':'); #else return path[0] == '/'; #endif } static string currentWorkingDirectory() { #ifdef _WIN32 const DWORD needed = GetCurrentDirectoryA(0, nullptr); if (needed == 0) throw runtime_error("cannot determine current directory"); vector<char> buffer(static_cast<size_t>(needed) + 1u); const DWORD written = GetCurrentDirectoryA(static_cast<DWORD>(buffer.size()), buffer.data()); if (written == 0 || written >= buffer.size()) throw runtime_error("cannot determine current directory"); return normalizePathSeparators(string(buffer.data(), written)); #else char buffer[PATH_MAX]; if (getcwd(buffer, sizeof(buffer)) == nullptr) throw runtime_error("cannot determine current directory"); return string(buffer); #endif } static string canonicalExistingPath(const string &input) { if (input.empty()) throw runtime_error("empty filesystem path"); #ifdef _WIN32 const DWORD fullNeeded = GetFullPathNameA(input.c_str(), 0, nullptr, nullptr); if (fullNeeded == 0) throw runtime_error("cannot make path absolute: " + input); vector<char> fullBuffer(static_cast<size_t>(fullNeeded) + 1u); const DWORD fullWritten = GetFullPathNameA( input.c_str(), static_cast<DWORD>(fullBuffer.size()), fullBuffer.data(), nullptr); if (fullWritten == 0 || fullWritten >= fullBuffer.size()) throw runtime_error("cannot make path absolute: " + input); const string absolutePath(fullBuffer.data(), fullWritten); HANDLE handle = CreateFileA(absolutePath.c_str(), FILE_READ_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr); if (handle == INVALID_HANDLE_VALUE) throw runtime_error("cannot resolve path: " + input + " (Windows error " + to_string(GetLastError()) + ")"); const DWORD finalNeeded = GetFinalPathNameByHandleA( handle, nullptr, 0, FILE_NAME_NORMALIZED | VOLUME_NAME_DOS); if (finalNeeded == 0) { const DWORD error = GetLastError(); CloseHandle(handle); throw runtime_error("cannot canonicalize path: " + input + " (Windows error " + to_string(error) + ")"); } vector<char> finalBuffer(static_cast<size_t>(finalNeeded) + 1u); const DWORD finalWritten = GetFinalPathNameByHandleA( handle, finalBuffer.data(), static_cast<DWORD>(finalBuffer.size()), FILE_NAME_NORMALIZED | VOLUME_NAME_DOS); const DWORD finalError = GetLastError(); CloseHandle(handle); if (finalWritten == 0 || finalWritten >= finalBuffer.size()) throw runtime_error("cannot canonicalize path: " + input + " (Windows error " + to_string(finalError) + ")"); string result(finalBuffer.data(), finalWritten); if (result.compare(0, 8, "\\\\?\\UNC\\") == 0) result = "\\\\" + result.substr(8); else if (result.compare(0, 4, "\\\\?\\") == 0) result.erase(0, 4); return normalizePathSeparators(move(result)); #else char resolved[PATH_MAX]; if (realpath(input.c_str(), resolved) == nullptr) throw runtime_error("cannot canonicalize path: " + input); return string(resolved); #endif } static bool pathIsDirectory(const string &path) noexcept { #ifdef _WIN32 const DWORD attributes = GetFileAttributesA(path.c_str()); return attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0; #else struct stat info {}; return stat(path.c_str(), &info) == 0 && S_ISDIR(info.st_mode); #endif } static bool pathIsRegularFile(const string &path) noexcept { #ifdef _WIN32 const DWORD attributes = GetFileAttributesA(path.c_str()); return attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0; #else struct stat info {}; return stat(path.c_str(), &info) == 0 && S_ISREG(info.st_mode); #endif } static string parentPath(const string &path) { const size_t separator = path.find_last_of("/\\"); if (separator == string::npos) return {}; if (separator == 0) return path.substr(0, 1); #ifdef _WIN32 if (separator == 2 && path.size() >= 3 && path[1] == ':') return path.substr(0, 3); #endif return path.substr(0, separator); } static string joinPath(const string &parent, const string &child) { if (parent.empty()) return child; if (parent.back() == '/' || parent.back() == '\\') return parent + child; return parent + '/' + child; } static bool pathWithin(const string &rootPath, const string &filePath) { string root = normalizePathSeparators(rootPath); string file = normalizePathSeparators(filePath); #ifdef _WIN32 root = lowerAscii(move(root)); file = lowerAscii(move(file)); #endif while (root.size() > 1 && root.back() == '/' && !(root.size() == 3 && root[1] == ':')) root.pop_back(); if (file == root) return true; if (root.empty()) return false; if (root.back() != '/') root.push_back('/'); return file.compare(0, root.size(), root) == 0; } static string readBoundedFile(const string &path, size_t limit, const char *kind) { if (!pathIsRegularFile(path)) throw runtime_error(string("cannot open ") + kind + " file: " + path); ifstream input(path.c_str(), ios::binary); if (!input) throw runtime_error(string("cannot open ") + kind + " file: " + path); string bytes; array<char, 65536> buffer{}; while (input) { input.read(buffer.data(), static_cast<streamsize>(buffer.size())); const streamsize count = input.gcount(); if (count > 0) { const size_t amount = static_cast<size_t>(count); if (bytes.size() > limit || amount > limit - bytes.size()) throw runtime_error(string(kind) + " file exceeds configured size limit: " + path); bytes.append(buffer.data(), amount); } } if (!input.eof()) throw runtime_error(string("failed reading ") + kind + " file: " + path); return bytes; } static string trimCopy(string s) { size_t a = 0, b = s.size(); while (a < b && isspace(u8(s[a]))) ++a; while (b > a && isspace(u8(s[b - 1]))) --b; return s.substr(a, b - a); } static bool parseIncludeLine(const string &line, string &name, KeywordCorrector &corrector, size_t lineNumber) { string s = trimCopy(line); size_t wordEnd = 0; while (wordEnd < s.size() && isIdentifierContinue(u8(s[wordEnd]))) ++wordEnd; if (wordEnd == 0) return false; size_t pathStart = wordEnd; while (pathStart < s.size() && isspace(u8(s[pathStart]))) ++pathStart; if (pathStart >= s.size() || s[pathStart] != '"') return false; const size_t quoteEnd = s.find('"', pathStart + 1); if (quoteEnd == string::npos) return false; size_t tail = quoteEnd + 1; while (tail < s.size() && isspace(u8(s[tail]))) ++tail; if (tail >= s.size() || s[tail] != ';') return false; ++tail; while (tail < s.size() && isspace(u8(s[tail]))) ++tail; if (tail != s.size()) return false; string keyword = s.substr(0, wordEnd); if (lowerAscii(keyword) != "include") { static const vector<string> expected = {"include"}; const size_t firstCharacter = line.find_first_not_of(" \t\r\n"); const size_t column = firstCharacter == string::npos ? 1 : firstCharacter + 1; if (!corrector.suggest(keyword, expected, lineNumber, column, true, true)) return false; s.replace(0, wordEnd, keyword); pathStart = keyword.size(); while (pathStart < s.size() && isspace(u8(s[pathStart]))) ++pathStart; } const size_t correctedQuoteEnd = s.find('"', pathStart + 1); if (correctedQuoteEnd == string::npos) return false; name = s.substr(pathStart + 1, correctedQuoteEnd - pathStart - 1); return !name.empty(); } [[maybe_unused]] static string expandIncludes(const string &sourcePath, const string &includeRootArg, KeywordCorrector &corrector) { string rootCandidate = includeRootArg.empty() ? parentPath(sourcePath) : includeRootArg; if (rootCandidate.empty()) rootCandidate = currentWorkingDirectory(); const string root = canonicalExistingPath(rootCandidate); if (!pathIsDirectory(root)) throw runtime_error("include root is not a directory"); const string first = canonicalExistingPath(sourcePath); if (!pathIsRegularFile(first)) throw runtime_error("source path is not a regular file"); if (!pathWithin(root, first)) throw runtime_error("source file is outside the permitted include root"); struct Frame { string path; string data; size_t offset = 0; size_t line = 1; }; vector<Frame> stack; StringSet active, visited; size_t total = 0, files = 0; auto pushFile = [&](const string &path) { const string key = normalizePathSeparators(path); if (active.find(key) != active.end()) throw runtime_error("cyclic include detected: " + key); if (visited.find(key) != visited.end()) return; if (files >= 128) throw runtime_error("include file-count limit (128) exceeded"); string data = readBoundedFile(key, MAX_SOURCE_BYTES, "source/include"); if (total > MAX_SOURCE_BYTES || data.size() > MAX_SOURCE_BYTES - total) throw runtime_error("combined source/include size limit exceeded"); total += data.size(); ++files; active.insert(key); visited.insert(key); stack.push_back({key, move(data), 0, 1}); }; pushFile(first); string output; output.reserve(min<size_t>(MAX_SOURCE_BYTES, total + 1024)); while (!stack.empty()) { Frame &frame = stack.back(); if (frame.offset >= frame.data.size()) { active.erase(frame.path); stack.pop_back(); continue; } const size_t start = frame.offset; const size_t newline = frame.data.find('\n', start); const size_t end = newline == string::npos ? frame.data.size() : newline; string line = frame.data.substr(start, end - start); frame.offset = newline == string::npos ? frame.data.size() : newline + 1; const size_t sourceLine = frame.line++; string includeName; if (parseIncludeLine(line, includeName, corrector, sourceLine)) { if (isAbsolutePathString(includeName)) throw runtime_error("absolute include paths are prohibited"); const string candidate = joinPath(parentPath(frame.path), includeName); const string resolved = canonicalExistingPath(candidate); if (!pathWithin(root, resolved)) throw runtime_error("include path escapes permitted root: " + includeName); if (!pathIsRegularFile(resolved)) throw runtime_error("include target is not a regular file: " + includeName); pushFile(resolved); } else { if (output.size() >= MAX_SOURCE_BYTES || line.size() >= MAX_SOURCE_BYTES - output.size()) throw runtime_error("expanded source exceeds size limit"); output.append(line); output.push_back('\n'); } } return output; } struct Options{ string sourcePath,dictionaryPath,nativeOutput; string includeRoot, runtimeSource; int depth=2; bool checkOnly=false,noCorrect=false,autoCorrect=false,help=false; uint64_t gas=DEFAULT_GAS; }; static uint64_t parseUnsigned(const string&s,const char*name){ if(s.empty()||s.size()>20) throw runtime_error(string("invalid ")+name); uint64_t x=0; for(char c:s){ if(!isAsciiDigit(u8(c))) throw runtime_error(string("invalid ")+name); const unsigned d=static_cast<unsigned>(c-'0'); if(x>(numeric_limits<uint64_t>::max()-d)/10) throw runtime_error(string(name)+" is too large"); x=x*10+d; } return x; } [[maybe_unused]] static Options parseArguments(int argc,char**argv){ Options o;for(int i=1;i<argc;++i){const string a=argv[i];auto value=[&](const char*name)->string{if(i+1>=argc)throw runtime_error(string("missing value after ")+name);return argv[++i];}; if(a=="--help"||a=="-h")o.help=true;else if(a=="--dictionary")o.dictionaryPath=value("--dictionary");else if(a=="--depth"){const uint64_t depth=parseUnsigned(value("--depth"),"dictionary depth");if(depth>16)throw runtime_error("--depth must be between 0 and 16");o.depth=static_cast<int>(depth);}else if(a=="--gas")o.gas=parseUnsigned(value("--gas"),"gas limit");else if(a=="--include-root")o.includeRoot=value("--include-root");else if(a=="--emit-cpp")o.nativeOutput=value("--emit-cpp");else if(a=="--runtime-source")o.runtimeSource=value("--runtime-source");else if(a=="--check")o.checkOnly=true;else if(a=="--no-correct")o.noCorrect=true;else if(a=="--auto-correct")o.autoCorrect=true;else if(!a.empty()&&a[0]=='-')throw runtime_error("unknown option: "+a);else if(o.sourcePath.empty())o.sourcePath=a;else throw runtime_error("only one source file may be supplied"); } if(o.help)return o; if(o.sourcePath.empty())throw runtime_error("usage: englishlike program.el [--dictionary file.json] [--depth N] [--gas N] [--include-root DIR] [--emit-cpp FILE] [--runtime-source FILE] [--check] [--no-correct] [--auto-correct]"); if(o.depth<0||o.depth>16) throw runtime_error("--depth must be between 0 and 16"); if(o.gas>1000000000000ULL) throw runtime_error("--gas must be 0 (unlimited) or at most 1000000000000"); return o; } // dictionary.cpp provides the default ChatIPC-format JSON payload by pointer // and length; --dictionary can supply a replacement JSON file. #ifndef ENGLISHLIKE_LIBRARY_ONLY static vector<DictionaryEntry> loadDictionary(const Options&o){ if (!o.dictionaryPath.empty()) { const string bytes=readBoundedFile(o.dictionaryPath,MAX_DICTIONARY_BYTES,"dictionary"); return JsonReader(string_view(bytes.data(),bytes.size())).parseDictionary(); } if (dictionary_json_len > MAX_DICTIONARY_BYTES) throw runtime_error("default dictionary exceeds configured size limit"); const string_view bytes(reinterpret_cast<const char *>(dictionary_json), static_cast<size_t>(dictionary_json_len)); return JsonReader(bytes).parseDictionary(); } #endif static void verifyProgram(const Program&p){ const size_t noId=numeric_limits<size_t>::max(); if(p.functions.empty()||p.functions[0].name!="$main")throw runtime_error("bytecode verifier: invalid program entry point"); if(p.functions.size()>MAX_FUNCTIONS||p.types.size()>MAX_TYPES||p.globals.size()>MAX_GLOBALS)throw runtime_error("bytecode verifier: program table exceeds configured limits"); for(size_t i=0;i<p.functions.size();++i){ const FunctionInfo&f=p.functions[i]; if(f.localNames.size()>MAX_LOCALS_PER_FUNCTION||f.parameters.size()>MAX_PARAMETERS)throw runtime_error("bytecode verifier: function metadata exceeds limits"); if(f.localTypes.size()!=f.localNames.size())throw runtime_error("bytecode verifier: local type table size mismatch"); if(f.ownerType!=noId&&f.ownerType>=p.types.size())throw runtime_error("bytecode verifier: invalid function owner type"); for(const Instruction&ins:f.code){ if(static_cast<unsigned>(ins.op)>static_cast<unsigned>(Op::Throw))throw runtime_error("bytecode verifier: invalid opcode"); switch(ins.op){ case Op::PushNumber:if(ins.a>=p.compileConstants.size())throw runtime_error("bytecode verifier: invalid numeric constant index");break; case Op::PushString:case Op::GetField:case Op::SetField:case Op::Call:case Op::CallMethod: if(ins.a>=p.strings.size())throw runtime_error("bytecode verifier: invalid string-pool index"); if((ins.op==Op::Call||ins.op==Op::CallMethod)&&ins.b>MAX_PARAMETERS)throw runtime_error("bytecode verifier: call argument count exceeds limit"); break; case Op::LoadLocal:case Op::StoreLocal:if(ins.a>=f.localNames.size())throw runtime_error("bytecode verifier: invalid local-variable index");break; case Op::LoadGlobal:case Op::StoreGlobal:if(ins.a>=p.globals.size())throw runtime_error("bytecode verifier: invalid global-variable index");break; case Op::PushBoolean:if(ins.a>1)throw runtime_error("bytecode verifier: invalid Boolean literal");break; case Op::NewObject:if(ins.a>=p.types.size()||p.types[ins.a].isInterface)throw runtime_error("bytecode verifier: invalid object type");break; case Op::NewArray:if(ins.b>MAX_ARRAY_ELEMENTS)throw runtime_error("bytecode verifier: array literal exceeds limit");break; case Op::NewMap:if(ins.b>MAX_ARRAY_ELEMENTS*2||ins.b%2!=0)throw runtime_error("bytecode verifier: invalid map constructor count");break; case Op::Jump:case Op::JumpIfFalse:if(ins.a>f.code.size())throw runtime_error("bytecode verifier: invalid jump target");break; case Op::PushHandler:if(ins.a>f.code.size()||ins.b>=f.localNames.size())throw runtime_error("bytecode verifier: invalid exception-handler metadata");break; case Op::Read:if((ins.b==0&&ins.a>=f.localNames.size())||(ins.b==1&&ins.a>=p.globals.size())||ins.b>1)throw runtime_error("bytecode verifier: invalid input destination");break; default:break; } } } for(size_t i=0;i<p.types.size();++i){ const TypeInfo&type=p.types[i]; if(type.base!=noId&&type.base>=p.types.size())throw runtime_error("bytecode verifier: invalid base type"); size_t ancestor=type.base,hops=0; while(ancestor!=noId){if(ancestor==i)throw runtime_error("bytecode verifier: cyclic class inheritance");if(ancestor>=p.types.size()||++hops>p.types.size())throw runtime_error("bytecode verifier: invalid inheritance chain");ancestor=p.types[ancestor].base;} for(const auto&field:type.fieldIds)if(field.second>=type.fields.size()||type.fields[field.second].name!=field.first)throw runtime_error("bytecode verifier: inconsistent field index"); for(const auto&method:type.methods)if(method.second>=p.functions.size()||p.functions[method.second].ownerType==noId)throw runtime_error("bytecode verifier: invalid method index"); for(size_t interfaceId:type.interfaces)if(interfaceId>=p.types.size()||!p.types[interfaceId].isInterface)throw runtime_error("bytecode verifier: invalid implemented interface"); } for(const auto&entry:p.functionIds)if(entry.second>=p.functions.size()||p.functions[entry.second].name!=entry.first)throw runtime_error("bytecode verifier: inconsistent function symbol table"); for(const auto&entry:p.globalIds)if(entry.second>=p.globals.size()||p.globals[entry.second]!=entry.first)throw runtime_error("bytecode verifier: inconsistent global symbol table"); } static string cppStringLiteral(string_view value){ string out="\""; for(unsigned char c:value){ if(c=='"')out+="\\\""; else if(c=='\\')out+="\\\\"; else if(c=='\n')out+="\\n"; else if(c=='\r')out+="\\r"; else if(c=='\t')out+="\\t"; else if(c>=32&&c<127)out.push_back(static_cast<char>(c)); else {out.push_back('\\');out.push_back(static_cast<char>('0'+((c>>6)&7)));out.push_back(static_cast<char>('0'+((c>>3)&7)));out.push_back(static_cast<char>('0'+(c&7)));} } out.push_back('"');return out; } static string cppValueExpression(const Value&value){ if(holds_alternative<BigNat>(value))return "Value(BigNat("+cppStringLiteral(get<BigNat>(value).toString())+"))"; if(holds_alternative<double>(value)){ostringstream out;out<<setprecision(17)<<get<double>(value);return "Value("+out.str()+")";} if(holds_alternative<bool>(value))return string("Value(")+(get<bool>(value)?"true":"false")+")"; if(holds_alternative<string>(value))return "Value(string("+cppStringLiteral(get<string>(value))+"))"; throw runtime_error("native C++ backend cannot serialize object-valued compile-time constants"); } [[maybe_unused]] static void emitNativeCpp(const Program&p,const string&outputPath,const string&runtimeSourceArg,uint64_t gas){ string runtimeSource = runtimeSourceArg.empty() ? string(__FILE__) : runtimeSourceArg; try { runtimeSource = canonicalExistingPath(runtimeSource); } catch (const exception &) { throw runtime_error("native C++ backend requires an accessible runtime source; pass --runtime-source EnglishLike.cpp"); } if (!pathIsRegularFile(runtimeSource)) throw runtime_error("native C++ backend runtime source is not a regular file"); string includePath=runtimeSource; string quotedInclude;quotedInclude.reserve(includePath.size()+8); for(char c:includePath){if(c=='"'||c=='\\')quotedInclude.push_back('\\');quotedInclude.push_back(c);} ostringstream out; out<<"// Generated by EnglishLike 2.2. Build with a C++17 compiler.\n" <<"#define ENGLISHLIKE_LIBRARY_ONLY 1\n#include \""<<quotedInclude<<"\"\n" <<"int main(){try{Program p;\n"; out<<"p.functions.resize("<<p.functions.size()<<");\n"; for(size_t i=0;i<p.functions.size();++i){ const FunctionInfo&f=p.functions[i];out<<"{auto&x=p.functions["<<i<<"];x.name="<<cppStringLiteral(f.name)<<";x.ownerType="<<f.ownerType<<"ULL;x.generic="<<(f.generic?"true":"false")<<";x.access=static_cast<AccessLevel>("<<static_cast<unsigned>(f.access)<<");x.returnType=static_cast<StaticType>("<<static_cast<unsigned>(f.returnType)<<");\n"; for(const string&v:f.parameters)out<<"x.parameters.push_back("<<cppStringLiteral(v)<<");\n"; for(const string&v:f.localNames)out<<"x.localNames.push_back("<<cppStringLiteral(v)<<");\n"; for(const auto&v:f.localIds)out<<"x.localIds.emplace("<<cppStringLiteral(v.first)<<","<<v.second<<"ULL);\n"; for(const auto&v:f.localObjectTypes)out<<"x.localObjectTypes.emplace("<<cppStringLiteral(v.first)<<","<<v.second<<"ULL);\n"; for(StaticType v:f.localTypes)out<<"x.localTypes.push_back(static_cast<StaticType>("<<static_cast<unsigned>(v)<<"));\n"; for(StaticType v:f.parameterTypes)out<<"x.parameterTypes.push_back(static_cast<StaticType>("<<static_cast<unsigned>(v)<<"));\n"; for(const string&v:f.parameterTypeNames)out<<"x.parameterTypeNames.push_back("<<cppStringLiteral(v)<<");\n"; for(const string&v:f.genericParameters)out<<"x.genericParameters.push_back("<<cppStringLiteral(v)<<");\n"; for(const Instruction&ins:f.code)out<<"x.code.push_back({static_cast<Op>("<<static_cast<unsigned>(ins.op)<<"),"<<ins.a<<"ULL,"<<ins.b<<"ULL});\n"; out<<"p.functionIds.emplace(x.name,"<<i<<");}\n"; } out<<"p.types.resize("<<p.types.size()<<");\n"; for(size_t i=0;i<p.types.size();++i){ const TypeInfo&t=p.types[i];out<<"{auto&x=p.types["<<i<<"];x.name="<<cppStringLiteral(t.name)<<";x.base="<<t.base<<"ULL;x.isClass="<<(t.isClass?"true":"false")<<";x.isInterface="<<(t.isInterface?"true":"false")<<";\n"; for(const FieldInfo&f:t.fields){out<<"{FieldInfo z;z.name="<<cppStringLiteral(f.name)<<";z.initial="<<cppValueExpression(f.initial)<<";z.access=static_cast<AccessLevel>("<<static_cast<unsigned>(f.access)<<");z.type=static_cast<StaticType>("<<static_cast<unsigned>(f.type)<<");z.declaringType="<<f.declaringType<<"ULL;x.fieldIds.emplace(z.name,x.fields.size());x.fields.push_back(move(z));}\n";} for(const auto&v:t.methods)out<<"x.methods.emplace("<<cppStringLiteral(v.first)<<","<<v.second<<"ULL);\n"; for(size_t v:t.interfaces)out<<"x.interfaces.push_back("<<v<<"ULL);\n"; for(const auto&v:t.requiredMethods)out<<"x.requiredMethods.emplace("<<cppStringLiteral(v.first)<<","<<v.second<<"ULL);\n"; out<<"p.typeIds.emplace(x.name,"<<i<<");}\n"; } for(const string&v:p.globals)out<<"p.globals.push_back("<<cppStringLiteral(v)<<");\n"; for(const auto&v:p.globalIds)out<<"p.globalIds.emplace("<<cppStringLiteral(v.first)<<","<<v.second<<"ULL);\n"; for(StaticType v:p.globalTypes)out<<"p.globalTypes.push_back(static_cast<StaticType>("<<static_cast<unsigned>(v)<<"));\n"; for(const auto&v:p.globalObjectTypes)out<<"p.globalObjectTypes.emplace("<<cppStringLiteral(v.first)<<","<<v.second<<"ULL);\n"; for(const string&v:p.strings)out<<"p.strings.push_back("<<cppStringLiteral(v)<<");\n"; for(size_t i=0;i<p.strings.size();++i)out<<"p.stringIds.emplace(p.strings["<<i<<"],"<<i<<"ULL);\n"; for(const Value&v:p.compileConstants)out<<"p.compileConstants.push_back("<<cppValueExpression(v)<<");\n"; for(const auto&v:p.constantIds)out<<"p.constantIds.emplace("<<cppStringLiteral(v.first)<<","<<v.second<<"ULL);\n"; for(const auto&v:p.genericSpecializations)out<<"p.genericSpecializations.emplace("<<cppStringLiteral(v.first)<<","<<v.second<<"ULL);\n"; out<<"verifyProgram(p);VirtualMachine vm(p,"<<gas<<"ULL);vm.run();return 0;}catch(const exception&e){cerr<<\"Native EnglishLike program: \"<<e.what()<<'\\n';return 1;}}\n"; const string generated=out.str(); constexpr size_t MAX_NATIVE_SOURCE_BYTES=128u*1024u*1024u; if(generated.size()>MAX_NATIVE_SOURCE_BYTES)throw runtime_error("generated C++ source exceeds 128 MiB safety limit"); ofstream file(outputPath.c_str(),ios::binary|ios::trunc);if(!file)throw runtime_error("cannot create generated C++ output: "+outputPath); file.write(generated.data(),static_cast<streamsize>(generated.size()));if(!file)throw runtime_error("failed writing generated C++ output"); cerr<<"Generated native-buildable C++ launcher: "<<outputPath<<" ("<<generated.size()<<" bytes)\n"; } } // namespace englishlike #ifndef ENGLISHLIKE_LIBRARY_ONLY using namespace englishlike; int main(int argc,char**argv){ try{ const Options options=parseArguments(argc,argv); if (options.help) { cout << "EnglishLike 2.2\n" << "Usage: englishlike program.el [options]\n" << " --dictionary FILE Optional JSON dictionary override; linked dictionary is default\n" << " --depth N Definition expansion depth (0..16; default 2)\n" << " --gas N VM instruction budget (default 10000000; 0 unlimited)\n" << " --include-root DIR Restrict include expansion to this directory\n" << " --emit-cpp FILE Emit native-buildable C++ launcher\n" << " --runtime-source FILE Runtime source for generated launcher\n" << " --check Compile and verify without executing\n" << " --no-correct Disable interactive keyword suggestions\n" << " --auto-correct Accept only high-confidence corrections\n"; return 0; } vector<DictionaryEntry> entries=loadDictionary(options); DefinitionModel definitions(move(entries)); KeywordCorrector corrector( definitions, options.depth, options.noCorrect, options.autoCorrect); const string source=expandIncludes( options.sourcePath, options.includeRoot, corrector); vector<Token>tokens=Lexer(source).scan(); Compiler compiler(move(tokens), corrector); Program program=compiler.compile();verifyProgram(program); cerr<<"EnglishLike 2.2: compiled "<<program.functions.size()<<" function(s), "<<program.types.size()<<" type(s), "<<program.globals.size()<<" global(s).\n"; if(!options.nativeOutput.empty())emitNativeCpp(program,options.nativeOutput,options.runtimeSource,options.gas); else if(!options.checkOnly)VirtualMachine(program,options.gas).run(); return 0; }catch(const exception&e){cerr<<"EnglishLike: "<<e.what()<<'\n';return 1;} } #endif // ENGLISHLIKE_LIBRARY_ONLY
// EnglishLike 2.2 — iterative C++17 compiler and bounded bytecode runtime.
// Features: natural-English declarations, grammar-position keyword correction,
// functions, classes/structures,
// methods, inheritance, objects, includes, compile-time constants, generic
// (type-erased) function declarations, ChatIPC-inspired typo assistance.
// No recursive C++ algorithms are used in the parser, JSON reader, VM, or
// definition expansion.
//
// Build: g++ -std=c++17 -O3 -DNDEBUG -Wall -Wextra -pedantic -fopenmp EnglishLike.cpp dictionary.cpp -o englishlike
// Run: ./englishlike program.el --depth 2 (linked dictionary.cpp is default)
#include <algorithm>
#include <array>
#include <atomic>
#include <cctype>
#include <cstdint>
#include <cstdlib>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <limits>
#include <memory>
#include <random>
#include <chrono>
#include <cstddef>
#include <cmath>
#include <cstdio>
#include <functional>
#include <iterator>
#include <optional>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <variant>
#include <vector>
#if defined(_WIN32)
// GetFinalPathNameByHandleA is declared by the Windows SDK headers only when
// the target API level is Vista (0x0600) or newer. Define the target before
// including windows.h; MinGW-w64 GCC 8.1 otherwise hides the declaration.
# if !defined(_WIN32_WINNT) || (_WIN32_WINNT < 0x0600)
# undef _WIN32_WINNT
# define _WIN32_WINNT 0x0600
# endif
# if !defined(WINVER) || (WINVER < 0x0600)
# undef WINVER
# define WINVER 0x0600
# endif
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# ifndef NOMINMAX
# define NOMINMAX
# endif
# include <windows.h>
#else
# include <limits.h>
# include <sys/stat.h>
# include <unistd.h>
#endif
using namespace std;
#ifndef ENGLISHLIKE_LIBRARY_ONLY
extern unsigned char dictionary_json[];
extern unsigned int dictionary_json_len;
#endif
namespace englishlike {
static constexpr size_t MAX_SOURCE_BYTES = 32u * 1024u * 1024u;
static constexpr size_t MAX_DICTIONARY_BYTES = 512u * 1024u * 1024u;
static constexpr size_t MAX_DICTIONARY_RECORDS = 1000000u;
static constexpr size_t MAX_DEFINITIONS_PER_RECORD = 4096u;
static constexpr size_t MAX_TOKENIZED_WORDS = 8192u;
static constexpr size_t MAX_DEFINITION_TOKENS_PER_RECORD = 4096u;
static constexpr size_t MAX_DEFINITION_CACHE_TOKENS = 2000000u;
static constexpr size_t MAX_FINGERPRINT_TOKENS = 8192u;
static constexpr size_t MAX_TOKENS = 1u * 1024u * 1024u;
static constexpr size_t MAX_JSON_STRING_BYTES = 1u * 1024u * 1024u;
static constexpr size_t MAX_IDENTIFIER_BYTES = 4096u;
static constexpr size_t MAX_INTEGER_DIGITS = 8192u;
static constexpr size_t MAX_ARRAY_ELEMENTS = 262144u;
static constexpr size_t MAX_DIAGNOSTICS = 256u;
static constexpr size_t MAX_GENERATED_SOURCE_BYTES = 4u * 1024u * 1024u;
static constexpr size_t MAX_RUNTIME_STACK = 256u * 1024u;
static constexpr size_t MAX_CALL_FRAMES = 256;
static constexpr size_t MAX_OBJECTS = 262144u;
static constexpr size_t MAX_RUNTIME_OBJECT_FIELDS = 1u * 1024u * 1024u;
static constexpr size_t MAX_RUNTIME_HEAP_BYTES = 128u * 1024u * 1024u;
static constexpr size_t RUNTIME_SLOT_OVERHEAD = 32u;
static constexpr size_t MAX_LOCALS_PER_FUNCTION = 8192u;
static constexpr size_t MAX_GLOBALS = 100000u;
static constexpr size_t MAX_FUNCTIONS = 16384u;
static constexpr size_t MAX_TYPES = 8192u;
static constexpr size_t MAX_FIELDS_PER_TYPE = 8192u;
static constexpr size_t MAX_PARAMETERS = 1024u;

Xet Storage Details

Size:
183 kB
·
Xet hash:
0900f1597052b3dcf5489ed6d236d836a53c33294d6eaff045d0537c90c12aca

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.