dssl / scripts /gen-native-vocab.mjs

Claude Sonnet 5.5

v0.16.3: native DSSL spec, and the seven bugs writing it found

9da4a9b about 14 hours ago

10.2 kB

#!/usr/bin/env node
/* gen-native-vocab.mjs: write the vocabulary the native crate and the spec appendices share.
 *
 *   node scripts/gen-native-vocab.mjs           write the files
 *   node scripts/gen-native-vocab.mjs --check   fail if a file on disk is out of date
 *
 * Outputs:
 *   native/data/vocabulary.json            every word list, in the engine's own order (order decides did-you-mean ties)
 *   native/spec/appendix-a-vocabulary.md   the small tables, in prose form
 *   native/spec/appendix-b-messages.md     every error and warning template found in the engine source
 *
 * The resolver keeps several tables private. Rather than touch the engine, this reads resolver.js as text,
 * appends one export line to a temporary copy, and imports that. Nothing in public/dssl is changed. */
import { readFileSync, writeFileSync, mkdirSync, mkdtempSync, existsSync } from 'node:fs';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { tmpdir } from 'node:os';
import { join, dirname } from 'node:path';

const root = fileURLToPath(new URL('../', import.meta.url));
const engineDir = join(root, 'public/dssl');
const check = process.argv.includes('--check');

async function loadResolverInternals() {
  let src = readFileSync(join(engineDir, 'resolver.js'), 'utf8');
  const base = pathToFileURL(engineDir).href + '/';
  src = src.replace(/from '\.\//g, `from '${base}`);
  src += '\nexport const __VOCAB = { TIME_PROPS, UNITLESS_FN, LENGTH_FN, COLOR_WORDS, FLUID_PROPS, PLACE_WORDS, SEMANTIC_NAMES: Object.keys(SEMANTIC) };\n';
  const dir = mkdtempSync(join(tmpdir(), 'dssl-vocab-'));
  const file = join(dir, 'resolver-vocab.mjs');
  writeFileSync(file, src);
  return import(pathToFileURL(file).href);
}

const R = await loadResolverInternals();
const P = await import(pathToFileURL(join(engineDir, 'parser.js')).href);
const C = await import(pathToFileURL(join(engineDir, 'color.js')).href);
const T = await import(pathToFileURL(join(engineDir, 'texture.js')).href);
const A = await import(pathToFileURL(join(engineDir, 'aura.js')).href);
const S = await import(pathToFileURL(join(engineDir, 'slider.js')).href);
const E = await import(pathToFileURL(join(engineDir, 'effects.js')).href);
const F = await import(pathToFileURL(join(engineDir, 'fonts.js')).href);

const V = R.__VOCAB;
const vocab = {
  _about: 'Generated by scripts/gen-native-vocab.mjs from public/dssl. Do not edit by hand. Array order is the engine\'s insertion order.',
  version: (/export const VERSION = '([^']+)'/.exec(readFileSync(join(engineDir, 'index.js'), 'utf8')) || [])[1] || 'unknown',
  stateWords: [...P.STATE_WORDS],
  pseudoElements: [...P.PSEUDO_ELEMENTS],
  vendorPseudoSource: P.VENDOR_PSEUDO.source,
  cssKeywords: [...R.CSS_KEYWORDS],
  knownProps: [...R.KNOWN_PROPS],
  unitlessProps: [...R.UNITLESS_PROPS],
  timeProps: [...V.TIME_PROPS],
  unitlessFns: [...V.UNITLESS_FN],
  lengthFns: [...V.LENGTH_FN],
  colorWords: [...V.COLOR_WORDS],
  fluidProps: [...V.FLUID_PROPS],
  placeWords: V.PLACE_WORDS,
  easings: R.EASINGS,
  defaultTokens: R.DEFAULT_TOKENS,
  semanticNames: V.SEMANTIC_NAMES,
  namedColors: C.NAMED,
  texture: { generators: T.GENERATORS, blendModes: T.BLEND_MODES, keys: T.TEXTURE_KEYS, presetNames: T.PRESET_NAMES, presets: T.PRESETS },
  aura: { fromWords: A.FROM_WORDS, shapes: A.SHAPES, dirWords: A.DIR_WORDS, blends: A.AURA_BLENDS, glowKeys: A.GLOW_KEYS, particleKeys: A.PARTICLE_KEYS, glowDefaults: A.GLOW_DEFAULTS, particleDefaults: A.PARTICLE_DEFAULTS, aroundNames: A.AROUND_NAMES },
  slider: { clauses: S.SLIDER_CLAUSES, thumbShapes: S.THUMB_SHAPES },
  effects: { glassWords: E.GLASS_WORDS, tiltWords: E.TILT_WORDS },
  fonts: { words: F.FONT_WORDS },
};

/* ---------------- appendix A ---------------- */

const code = (x) => '`' + x + '`';
const list = (a) => a.map(code).join(', ');
const table = (head, rows) => [`| ${head.join(' | ')} |`, `|${head.map(() => '---').join('|')}|`, ...rows.map((r) => `| ${r.join(' | ')} |`)].join('\n');
const esc = (s) => String(s).replace(/\|/g, '\\|');

function appendixA() {
  const o = [];
  o.push('# Appendix A: vocabulary', '',
    'Generated by `scripts/gen-native-vocab.mjs` from the engine. Not edited by hand.',
    `The full word lists, in the engine's own order, are in \`native/data/vocabulary.json\`.`, '');
  o.push('## A.1 State words', '', list(vocab.stateWords), '');
  o.push('## A.2 Pseudo-elements', '', list(vocab.pseudoElements), '',
    `A vendor pseudo-element name matches \`${vocab.vendorPseudoSource}\`.`, '');
  o.push('## A.3 Unitless and time properties', '',
    'Unitless (a bare number stays bare):', '', list(vocab.unitlessProps), '',
    'Time (a bare number is milliseconds):', '', list(vocab.timeProps), '');
  o.push('## A.4 Functions', '',
    'Unitless functions (numbers inside stay bare):', '', list(vocab.unitlessFns), '',
    'Length functions (bare numbers inside are px):', '', list(vocab.lengthFns), '');
  o.push('## A.5 Fluid properties', '', list(vocab.fluidProps), '');
  o.push('## A.6 Place words', '', table(['word', 'cross', 'main'], Object.entries(vocab.placeWords).map(([k, v]) => [code(k), code(v[0]), code(v[1])])), '');
  o.push('## A.7 Easings', '', table(['name', 'CSS'], Object.entries(vocab.easings).map(([k, v]) => [code(k), code(v)])), '');
  o.push('## A.8 Built-in tokens', '', table(['key', 'value'], Object.entries(vocab.defaultTokens).map(([k, v]) => [code(k), code(esc(v))])), '');
  o.push('## A.9 Semantic properties', '', list(vocab.semanticNames), '');
  o.push('## A.10 Big sets', '', table(['set', 'count', 'where'], [
    ['CSS keywords', vocab.cssKeywords.length, '`cssKeywords` in the data file'],
    ['known CSS properties', vocab.knownProps.length, '`knownProps`'],
    ['named colors', Object.keys(vocab.namedColors).length, '`namedColors`'],
  ]), '', 'Color words allowed as gradient stops:', '', list(vocab.colorWords), '');
  o.push('## A.11 Texture', '',
    `Generators: ${list(vocab.texture.generators)}`, '',
    `Blend modes: ${list(vocab.texture.blendModes)}`, '',
    `Presets (${vocab.texture.presetNames.length}): ${list(vocab.texture.presetNames)}`, '',
    'Settings:', '',
    table(['key', 'kind', 'values', 'range', 'what it does'], Object.entries(vocab.texture.keys).map(([k, d]) => [
      code(k), d.kind, d.words ?? '', d.values ? list(d.values) : (d.min !== undefined ? `${d.min} to ${d.max}` : ''), esc(d.doc || ''),
    ])), '');
  o.push('## A.12 Aura', '',
    `From words: ${list(vocab.aura.fromWords)}`, '',
    `Shapes: ${list(vocab.aura.shapes)}`, '',
    `Direction words: ${list(Object.keys(vocab.aura.dirWords))}`, '',
    `Blends: ${list(vocab.aura.blends)}`, '',
    `Around shapes: ${list(vocab.aura.aroundNames)}`, '',
    'Glow settings:', '',
    table(['key', 'kind', 'range', 'what it does'], Object.entries(vocab.aura.glowKeys).map(([k, d]) => [code(k), d.kind, d.min !== undefined ? `${d.min} to ${d.max}` : (d.values ? list(d.values) : ''), esc(d.doc || '')])), '',
    `Glow defaults: ${code(JSON.stringify(vocab.aura.glowDefaults))}`, '',
    'Particle settings:', '',
    table(['key', 'kind', 'range', 'what it does'], Object.entries(vocab.aura.particleKeys).map(([k, d]) => [code(k), d.kind, d.min !== undefined ? `${d.min} to ${d.max}` : (d.values ? list(d.values) : ''), esc(d.doc || '')])), '',
    `Particle defaults: ${code(JSON.stringify(vocab.aura.particleDefaults))}`, '');
  o.push('## A.13 Slider', '', table(['clause', 'form'], Object.entries(vocab.slider.clauses).map(([k, v]) => [code(k), code(esc(v))])), '',
    `Thumb shapes: ${list(vocab.slider.thumbShapes)}`, '');
  o.push('## A.14 Glass and tilt', '',
    table(['word', 'meaning'], [...Object.entries(vocab.effects.glassWords), ...Object.entries(vocab.effects.tiltWords)].map(([k, v]) => [code(k), esc(v)])), '');
  o.push('## A.15 Fonts', '', table(['word', 'meaning'], Object.entries(vocab.fonts.words).map(([k, v]) => [code(k), esc(v)])), '');
  return o.join('\n') + '\n';
}

/* ---------------- appendix B ---------------- */

function appendixB() {
  const files = ['parser.js', 'resolver.js', 'texture.js', 'aura.js', 'slider.js', 'effects.js', 'fonts.js', 'lint.js'];
  const re = /(?:\bmessage:\s*|\berr\(\s*\w+\s*,\s*|\b(?:bad|hooks\.error)\(\s*)(`(?:[^`\\]|\\.)*`|'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*")/g;
  const o = ['# Appendix B: message templates', '',
    'Generated by `scripts/gen-native-vocab.mjs`. Every error and warning template found in the engine',
    'source, by file and line. `${...}` marks a value filled in at run time. The extraction is by pattern,',
    'so it can include a string that is not a message and can miss one built in an unusual way. It is a',
    'checklist for the corpus, not the definition: the definition is in the chapters.', ''];
  for (const f of files) {
    const text = readFileSync(join(engineDir, f), 'utf8');
    const seen = new Set();
    const rows = [];
    for (const m of text.matchAll(re)) {
      const raw = m[1].slice(1, -1).replace(/\\n/g, ' ').replace(/\\(['"`])/g, '$1');
      if (raw.length < 6 || seen.has(raw)) continue;
      seen.add(raw);
      const line = text.slice(0, m.index).split('\n').length;
      rows.push([String(line), code(esc(raw).replace(/`/g, "'"))]);
    }
    o.push(`## ${f}`, '', table(['line', 'template'], rows), '');
  }
  return o.join('\n') + '\n';
}

/* ---------------- write or check ---------------- */

const outputs = [
  ['native/data/vocabulary.json', JSON.stringify(vocab, null, 1) + '\n'],
  ['native/spec/appendix-a-vocabulary.md', appendixA()],
  ['native/spec/appendix-b-messages.md', appendixB()],
];

let stale = 0;
for (const [rel, text] of outputs) {
  const path = join(root, rel);
  if (check) {
    const now = existsSync(path) ? readFileSync(path, 'utf8') : null;
    if (now !== text) { console.error('STALE ' + rel + ' (run node scripts/gen-native-vocab.mjs)'); stale++; }
    continue;
  }
  mkdirSync(dirname(path), { recursive: true });
  writeFileSync(path, text);
  console.log('wrote ' + rel + ' (' + text.length + ' bytes)');
}
if (check) {
  if (stale) process.exit(1);
  console.log('vocabulary files are up to date');
}

	#!/usr/bin/env node
	/* gen-native-vocab.mjs: write the vocabulary the native crate and the spec appendices share.
	*
	* node scripts/gen-native-vocab.mjs write the files
	* node scripts/gen-native-vocab.mjs --check fail if a file on disk is out of date
	*
	* Outputs:
	* native/data/vocabulary.json every word list, in the engine's own order (order decides did-you-mean ties)
	* native/spec/appendix-a-vocabulary.md the small tables, in prose form
	* native/spec/appendix-b-messages.md every error and warning template found in the engine source
	*
	* The resolver keeps several tables private. Rather than touch the engine, this reads resolver.js as text,
	* appends one export line to a temporary copy, and imports that. Nothing in public/dssl is changed. */
	import { readFileSync, writeFileSync, mkdirSync, mkdtempSync, existsSync } from 'node:fs';
	import { fileURLToPath, pathToFileURL } from 'node:url';
	import { tmpdir } from 'node:os';
	import { join, dirname } from 'node:path';

	const root = fileURLToPath(new URL('../', import.meta.url));
	const engineDir = join(root, 'public/dssl');
	const check = process.argv.includes('--check');

	async function loadResolverInternals() {
	let src = readFileSync(join(engineDir, 'resolver.js'), 'utf8');
	const base = pathToFileURL(engineDir).href + '/';
	src = src.replace(/from '\.\//g, `from '${base}`);
	src += '\nexport const __VOCAB = { TIME_PROPS, UNITLESS_FN, LENGTH_FN, COLOR_WORDS, FLUID_PROPS, PLACE_WORDS, SEMANTIC_NAMES: Object.keys(SEMANTIC) };\n';
	const dir = mkdtempSync(join(tmpdir(), 'dssl-vocab-'));
	const file = join(dir, 'resolver-vocab.mjs');
	writeFileSync(file, src);
	return import(pathToFileURL(file).href);
	}

	const R = await loadResolverInternals();
	const P = await import(pathToFileURL(join(engineDir, 'parser.js')).href);
	const C = await import(pathToFileURL(join(engineDir, 'color.js')).href);
	const T = await import(pathToFileURL(join(engineDir, 'texture.js')).href);
	const A = await import(pathToFileURL(join(engineDir, 'aura.js')).href);
	const S = await import(pathToFileURL(join(engineDir, 'slider.js')).href);
	const E = await import(pathToFileURL(join(engineDir, 'effects.js')).href);
	const F = await import(pathToFileURL(join(engineDir, 'fonts.js')).href);

	const V = R.__VOCAB;
	const vocab = {
	_about: 'Generated by scripts/gen-native-vocab.mjs from public/dssl. Do not edit by hand. Array order is the engine\'s insertion order.',
	version: (/export const VERSION = '([^']+)'/.exec(readFileSync(join(engineDir, 'index.js'), 'utf8')) \|\| [])[1] \|\| 'unknown',
	stateWords: [...P.STATE_WORDS],
	pseudoElements: [...P.PSEUDO_ELEMENTS],
	vendorPseudoSource: P.VENDOR_PSEUDO.source,
	cssKeywords: [...R.CSS_KEYWORDS],
	knownProps: [...R.KNOWN_PROPS],
	unitlessProps: [...R.UNITLESS_PROPS],
	timeProps: [...V.TIME_PROPS],
	unitlessFns: [...V.UNITLESS_FN],
	lengthFns: [...V.LENGTH_FN],
	colorWords: [...V.COLOR_WORDS],
	fluidProps: [...V.FLUID_PROPS],
	placeWords: V.PLACE_WORDS,
	easings: R.EASINGS,
	defaultTokens: R.DEFAULT_TOKENS,
	semanticNames: V.SEMANTIC_NAMES,
	namedColors: C.NAMED,
	texture: { generators: T.GENERATORS, blendModes: T.BLEND_MODES, keys: T.TEXTURE_KEYS, presetNames: T.PRESET_NAMES, presets: T.PRESETS },
	aura: { fromWords: A.FROM_WORDS, shapes: A.SHAPES, dirWords: A.DIR_WORDS, blends: A.AURA_BLENDS, glowKeys: A.GLOW_KEYS, particleKeys: A.PARTICLE_KEYS, glowDefaults: A.GLOW_DEFAULTS, particleDefaults: A.PARTICLE_DEFAULTS, aroundNames: A.AROUND_NAMES },
	slider: { clauses: S.SLIDER_CLAUSES, thumbShapes: S.THUMB_SHAPES },
	effects: { glassWords: E.GLASS_WORDS, tiltWords: E.TILT_WORDS },
	fonts: { words: F.FONT_WORDS },
	};

	/* ---------------- appendix A ---------------- */

	const code = (x) => '`' + x + '`';
	const list = (a) => a.map(code).join(', ');
	const table = (head, rows) => [`\| ${head.join(' \| ')} \|`, `\|${head.map(() => '---').join('\|')}\|`, ...rows.map((r) => `\| ${r.join(' \| ')} \|`)].join('\n');
	const esc = (s) => String(s).replace(/\\|/g, '\\\|');

	function appendixA() {
	const o = [];
	o.push('# Appendix A: vocabulary', '',
	'Generated by `scripts/gen-native-vocab.mjs` from the engine. Not edited by hand.',
	`The full word lists, in the engine's own order, are in \`native/data/vocabulary.json\`.`, '');
	o.push('## A.1 State words', '', list(vocab.stateWords), '');
	o.push('## A.2 Pseudo-elements', '', list(vocab.pseudoElements), '',
	`A vendor pseudo-element name matches \`${vocab.vendorPseudoSource}\`.`, '');
	o.push('## A.3 Unitless and time properties', '',
	'Unitless (a bare number stays bare):', '', list(vocab.unitlessProps), '',
	'Time (a bare number is milliseconds):', '', list(vocab.timeProps), '');
	o.push('## A.4 Functions', '',
	'Unitless functions (numbers inside stay bare):', '', list(vocab.unitlessFns), '',
	'Length functions (bare numbers inside are px):', '', list(vocab.lengthFns), '');
	o.push('## A.5 Fluid properties', '', list(vocab.fluidProps), '');
	o.push('## A.6 Place words', '', table(['word', 'cross', 'main'], Object.entries(vocab.placeWords).map(([k, v]) => [code(k), code(v[0]), code(v[1])])), '');
	o.push('## A.7 Easings', '', table(['name', 'CSS'], Object.entries(vocab.easings).map(([k, v]) => [code(k), code(v)])), '');
	o.push('## A.8 Built-in tokens', '', table(['key', 'value'], Object.entries(vocab.defaultTokens).map(([k, v]) => [code(k), code(esc(v))])), '');
	o.push('## A.9 Semantic properties', '', list(vocab.semanticNames), '');
	o.push('## A.10 Big sets', '', table(['set', 'count', 'where'], [
	['CSS keywords', vocab.cssKeywords.length, '`cssKeywords` in the data file'],
	['known CSS properties', vocab.knownProps.length, '`knownProps`'],
	['named colors', Object.keys(vocab.namedColors).length, '`namedColors`'],
	]), '', 'Color words allowed as gradient stops:', '', list(vocab.colorWords), '');
	o.push('## A.11 Texture', '',
	`Generators: ${list(vocab.texture.generators)}`, '',
	`Blend modes: ${list(vocab.texture.blendModes)}`, '',
	`Presets (${vocab.texture.presetNames.length}): ${list(vocab.texture.presetNames)}`, '',
	'Settings:', '',
	table(['key', 'kind', 'values', 'range', 'what it does'], Object.entries(vocab.texture.keys).map(([k, d]) => [
	code(k), d.kind, d.words ?? '', d.values ? list(d.values) : (d.min !== undefined ? `${d.min} to ${d.max}` : ''), esc(d.doc \|\| ''),
	])), '');
	o.push('## A.12 Aura', '',
	`From words: ${list(vocab.aura.fromWords)}`, '',
	`Shapes: ${list(vocab.aura.shapes)}`, '',
	`Direction words: ${list(Object.keys(vocab.aura.dirWords))}`, '',
	`Blends: ${list(vocab.aura.blends)}`, '',
	`Around shapes: ${list(vocab.aura.aroundNames)}`, '',
	'Glow settings:', '',
	table(['key', 'kind', 'range', 'what it does'], Object.entries(vocab.aura.glowKeys).map(([k, d]) => [code(k), d.kind, d.min !== undefined ? `${d.min} to ${d.max}` : (d.values ? list(d.values) : ''), esc(d.doc \|\| '')])), '',
	`Glow defaults: ${code(JSON.stringify(vocab.aura.glowDefaults))}`, '',
	'Particle settings:', '',
	table(['key', 'kind', 'range', 'what it does'], Object.entries(vocab.aura.particleKeys).map(([k, d]) => [code(k), d.kind, d.min !== undefined ? `${d.min} to ${d.max}` : (d.values ? list(d.values) : ''), esc(d.doc \|\| '')])), '',
	`Particle defaults: ${code(JSON.stringify(vocab.aura.particleDefaults))}`, '');
	o.push('## A.13 Slider', '', table(['clause', 'form'], Object.entries(vocab.slider.clauses).map(([k, v]) => [code(k), code(esc(v))])), '',
	`Thumb shapes: ${list(vocab.slider.thumbShapes)}`, '');
	o.push('## A.14 Glass and tilt', '',
	table(['word', 'meaning'], [...Object.entries(vocab.effects.glassWords), ...Object.entries(vocab.effects.tiltWords)].map(([k, v]) => [code(k), esc(v)])), '');
	o.push('## A.15 Fonts', '', table(['word', 'meaning'], Object.entries(vocab.fonts.words).map(([k, v]) => [code(k), esc(v)])), '');
	return o.join('\n') + '\n';
	}

	/* ---------------- appendix B ---------------- */

	function appendixB() {
	const files = ['parser.js', 'resolver.js', 'texture.js', 'aura.js', 'slider.js', 'effects.js', 'fonts.js', 'lint.js'];
	const re = /(?:\bmessage:\s\|\berr\(\s\w+\s,\s\|\b(?:bad\|hooks\.error)\(\s)(`(?:[^`\\]\|\\.)`\|'(?:[^'\\]\|\\.)'\|"(?:[^"\\]\|\\.)")/g;
	const o = ['# Appendix B: message templates', '',
	'Generated by `scripts/gen-native-vocab.mjs`. Every error and warning template found in the engine',
	'source, by file and line. `${...}` marks a value filled in at run time. The extraction is by pattern,',
	'so it can include a string that is not a message and can miss one built in an unusual way. It is a',
	'checklist for the corpus, not the definition: the definition is in the chapters.', ''];
	for (const f of files) {
	const text = readFileSync(join(engineDir, f), 'utf8');
	const seen = new Set();
	const rows = [];
	for (const m of text.matchAll(re)) {
	const raw = m[1].slice(1, -1).replace(/\\n/g, ' ').replace(/\\(['"`])/g, '$1');
	if (raw.length < 6 \|\| seen.has(raw)) continue;
	seen.add(raw);
	const line = text.slice(0, m.index).split('\n').length;
	rows.push([String(line), code(esc(raw).replace(/`/g, "'"))]);
	}
	o.push(`## ${f}`, '', table(['line', 'template'], rows), '');
	}
	return o.join('\n') + '\n';
	}

	/* ---------------- write or check ---------------- */

	const outputs = [
	['native/data/vocabulary.json', JSON.stringify(vocab, null, 1) + '\n'],
	['native/spec/appendix-a-vocabulary.md', appendixA()],
	['native/spec/appendix-b-messages.md', appendixB()],
	];

	let stale = 0;
	for (const [rel, text] of outputs) {
	const path = join(root, rel);
	if (check) {
	const now = existsSync(path) ? readFileSync(path, 'utf8') : null;
	if (now !== text) { console.error('STALE ' + rel + ' (run node scripts/gen-native-vocab.mjs)'); stale++; }
	continue;
	}
	mkdirSync(dirname(path), { recursive: true });
	writeFileSync(path, text);
	console.log('wrote ' + rel + ' (' + text.length + ' bytes)');
	}
	if (check) {
	if (stale) process.exit(1);
	console.log('vocabulary files are up to date');
	}