Spaces:
Paused
Paused
| /** | |
| * Licensed to the Apache Software Foundation (ASF) under one | |
| * or more contributor license agreements. See the NOTICE file | |
| * distributed with this work for additional information | |
| * regarding copyright ownership. The ASF licenses this file | |
| * to you under the Apache License, Version 2.0 (the | |
| * "License"); you may not use this file except in compliance | |
| * with the License. You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, | |
| * software distributed under the License is distributed on an | |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| * KIND, either express or implied. See the License for the | |
| * specific language governing permissions and limitations | |
| * under the License. | |
| */ | |
| /** | |
| * Smart generator wrapper: only runs generators whose input files have changed. | |
| * | |
| * Computes a hash of each generator's input files (stories, engine specs, | |
| * openapi.json, and the generator scripts themselves). Compares against a | |
| * stored cache. Skips generators whose inputs and outputs are unchanged. | |
| * | |
| * Usage: | |
| * node scripts/generate-if-changed.mjs # smart mode (default) | |
| * node scripts/generate-if-changed.mjs --force # force regenerate all | |
| */ | |
| import { createHash } from 'crypto'; | |
| import { execSync, spawn } from 'child_process'; | |
| import fs from 'fs'; | |
| import path from 'path'; | |
| import { fileURLToPath } from 'url'; | |
| const __filename = fileURLToPath(import.meta.url); | |
| const __dirname = path.dirname(__filename); | |
| const DOCS_DIR = path.resolve(__dirname, '..'); | |
| const ROOT_DIR = path.resolve(DOCS_DIR, '..'); | |
| const CACHE_FILE = path.join(DOCS_DIR, '.docusaurus', 'generator-hashes.json'); | |
| const FORCE = process.argv.includes('--force'); | |
| // Ensure local node_modules/.bin is on PATH (needed for docusaurus CLI) | |
| const localBin = path.join(DOCS_DIR, 'node_modules', '.bin'); | |
| process.env.PATH = `${localBin}${path.delimiter}${process.env.PATH}`; | |
| // --------------------------------------------------------------------------- | |
| // Generator definitions | |
| // --------------------------------------------------------------------------- | |
| const GENERATORS = [ | |
| { | |
| name: 'superset-components', | |
| command: 'node scripts/generate-superset-components.mjs', | |
| inputs: [ | |
| { | |
| type: 'glob', | |
| base: path.join( | |
| ROOT_DIR, | |
| 'superset-frontend/packages/superset-ui-core/src/components', | |
| ), | |
| pattern: '**/*.stories.tsx', | |
| }, | |
| { | |
| type: 'glob', | |
| base: path.join( | |
| ROOT_DIR, | |
| 'superset-frontend/packages/superset-core/src', | |
| ), | |
| pattern: '**/*.stories.tsx', | |
| }, | |
| { | |
| type: 'file', | |
| path: path.join(DOCS_DIR, 'scripts/generate-superset-components.mjs'), | |
| }, | |
| { | |
| type: 'file', | |
| path: path.join(DOCS_DIR, 'src/components/StorybookWrapper.jsx'), | |
| }, | |
| ], | |
| outputs: [ | |
| path.join(DOCS_DIR, 'developer_docs/components/index.mdx'), | |
| path.join(DOCS_DIR, 'static/data/components.json'), | |
| path.join(DOCS_DIR, 'src/types/apache-superset-core/index.d.ts'), | |
| ], | |
| }, | |
| { | |
| name: 'database-docs', | |
| command: 'node scripts/generate-database-docs.mjs', | |
| inputs: [ | |
| { | |
| type: 'glob', | |
| base: path.join(ROOT_DIR, 'superset/db_engine_specs'), | |
| pattern: '**/*.py', | |
| }, | |
| { | |
| type: 'file', | |
| path: path.join(DOCS_DIR, 'scripts/generate-database-docs.mjs'), | |
| }, | |
| ], | |
| outputs: [ | |
| path.join(DOCS_DIR, 'src/data/databases.json'), | |
| path.join(DOCS_DIR, 'docs/databases/supported'), | |
| ], | |
| }, | |
| { | |
| name: 'api-docs', | |
| command: | |
| 'python3 scripts/fix-openapi-spec.py && docusaurus gen-api-docs superset && node scripts/convert-api-sidebar.mjs && node scripts/generate-api-index.mjs && node scripts/generate-api-tag-pages.mjs', | |
| inputs: [ | |
| { | |
| type: 'file', | |
| path: path.join(DOCS_DIR, 'static/resources/openapi.json'), | |
| }, | |
| { | |
| type: 'file', | |
| path: path.join(DOCS_DIR, 'scripts/fix-openapi-spec.py'), | |
| }, | |
| { | |
| type: 'file', | |
| path: path.join(DOCS_DIR, 'scripts/convert-api-sidebar.mjs'), | |
| }, | |
| { | |
| type: 'file', | |
| path: path.join(DOCS_DIR, 'scripts/generate-api-index.mjs'), | |
| }, | |
| { | |
| type: 'file', | |
| path: path.join(DOCS_DIR, 'scripts/generate-api-tag-pages.mjs'), | |
| }, | |
| ], | |
| outputs: [path.join(DOCS_DIR, 'docs/api.mdx')], | |
| }, | |
| ]; | |
| // --------------------------------------------------------------------------- | |
| // Hashing utilities | |
| // --------------------------------------------------------------------------- | |
| function walkDir(dir, pattern) { | |
| const results = []; | |
| if (!fs.existsSync(dir)) return results; | |
| const regex = globToRegex(pattern); | |
| function walk(currentDir) { | |
| for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) { | |
| const fullPath = path.join(currentDir, entry.name); | |
| if (entry.isDirectory()) { | |
| if (entry.name === 'node_modules' || entry.name === '__pycache__') | |
| continue; | |
| walk(fullPath); | |
| } else { | |
| // Normalize to forward slashes so glob patterns work on all platforms | |
| const relativePath = path | |
| .relative(dir, fullPath) | |
| .split(path.sep) | |
| .join('/'); | |
| if (regex.test(relativePath)) { | |
| results.push(fullPath); | |
| } | |
| } | |
| } | |
| } | |
| walk(dir); | |
| return results.sort(); | |
| } | |
| function globToRegex(pattern) { | |
| // Simple glob-to-regex: ** matches any path, * matches anything except / | |
| const escaped = pattern | |
| .replace(/[.+^${}()|[\]\\]/g, '\\$&') | |
| .replace(/\*\*/g, '<<<GLOBSTAR>>>') | |
| .replace(/\*/g, '[^/]*') | |
| .replace(/<<<GLOBSTAR>>>/g, '.*'); | |
| return new RegExp(`^${escaped}$`); | |
| } | |
| function hashFile(filePath) { | |
| if (!fs.existsSync(filePath)) return 'missing'; | |
| const stat = fs.statSync(filePath); | |
| // Use mtime + size for speed (avoids reading file contents) | |
| return `${stat.mtimeMs}:${stat.size}`; | |
| } | |
| function computeInputHash(inputs) { | |
| const hash = createHash('md5'); | |
| for (const input of inputs) { | |
| if (input.type === 'file') { | |
| hash.update(`file:${input.path}:${hashFile(input.path)}\n`); | |
| } else if (input.type === 'glob') { | |
| const files = walkDir(input.base, input.pattern); | |
| hash.update( | |
| `glob:${input.base}:${input.pattern}:count=${files.length}\n`, | |
| ); | |
| for (const file of files) { | |
| hash.update(` ${path.relative(input.base, file)}:${hashFile(file)}\n`); | |
| } | |
| } | |
| } | |
| return hash.digest('hex'); | |
| } | |
| function outputsExist(outputs) { | |
| return outputs.every(p => fs.existsSync(p)); | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Cache management | |
| // --------------------------------------------------------------------------- | |
| function loadCache() { | |
| try { | |
| if (fs.existsSync(CACHE_FILE)) { | |
| return JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); | |
| } | |
| } catch { | |
| // Corrupted cache — start fresh | |
| } | |
| return {}; | |
| } | |
| function saveCache(cache) { | |
| const dir = path.dirname(CACHE_FILE); | |
| if (!fs.existsSync(dir)) { | |
| fs.mkdirSync(dir, { recursive: true }); | |
| } | |
| fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2)); | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Main | |
| // --------------------------------------------------------------------------- | |
| async function main() { | |
| const cache = loadCache(); | |
| const updatedCache = { ...cache }; | |
| let skipped = 0; | |
| let ran = 0; | |
| // First pass: determine which generators need to run | |
| // Run independent generators (superset-components, database-docs) in | |
| // parallel, then api-docs sequentially (it depends on docusaurus CLI | |
| // being available, not on other generators). | |
| const independent = GENERATORS.filter(g => g.name !== 'api-docs'); | |
| const sequential = GENERATORS.filter(g => g.name === 'api-docs'); | |
| // Check and run independent generators in parallel | |
| const parallelPromises = independent.map(gen => { | |
| const currentHash = computeInputHash(gen.inputs); | |
| const cachedHash = cache[gen.name]; | |
| const hasOutputs = outputsExist(gen.outputs); | |
| if (!FORCE && currentHash === cachedHash && hasOutputs) { | |
| console.log(` ✓ ${gen.name} — no changes, skipping`); | |
| skipped++; | |
| return Promise.resolve(); | |
| } | |
| const reason = FORCE | |
| ? 'forced' | |
| : !hasOutputs | |
| ? 'output missing' | |
| : 'inputs changed'; | |
| console.log(` ↻ ${gen.name} — ${reason}, regenerating...`); | |
| ran++; | |
| return new Promise((resolve, reject) => { | |
| const child = spawn('sh', ['-c', gen.command], { | |
| cwd: DOCS_DIR, | |
| stdio: 'inherit', | |
| env: process.env, | |
| }); | |
| child.on('close', code => { | |
| if (code === 0) { | |
| updatedCache[gen.name] = currentHash; | |
| resolve(); | |
| } else { | |
| console.error(` ✗ ${gen.name} failed (exit ${code})`); | |
| reject(new Error(`${gen.name} failed with exit code ${code}`)); | |
| } | |
| }); | |
| child.on('error', err => { | |
| console.error(` ✗ ${gen.name} failed to start`); | |
| reject(err); | |
| }); | |
| }); | |
| }); | |
| await Promise.all(parallelPromises); | |
| // Run sequential generators (api-docs) | |
| for (const gen of sequential) { | |
| const currentHash = computeInputHash(gen.inputs); | |
| const cachedHash = cache[gen.name]; | |
| const hasOutputs = outputsExist(gen.outputs); | |
| if (!FORCE && currentHash === cachedHash && hasOutputs) { | |
| console.log(` ✓ ${gen.name} — no changes, skipping`); | |
| skipped++; | |
| continue; | |
| } | |
| const reason = FORCE | |
| ? 'forced' | |
| : !hasOutputs | |
| ? 'output missing' | |
| : 'inputs changed'; | |
| console.log(` ↻ ${gen.name} — ${reason}, regenerating...`); | |
| ran++; | |
| try { | |
| execSync(gen.command, { | |
| cwd: DOCS_DIR, | |
| stdio: 'inherit', | |
| timeout: 300_000, | |
| }); | |
| updatedCache[gen.name] = currentHash; | |
| } catch (err) { | |
| console.error(` ✗ ${gen.name} failed`); | |
| throw err; | |
| } | |
| } | |
| saveCache(updatedCache); | |
| if (ran === 0) { | |
| console.log(`\nAll ${skipped} generators up-to-date — nothing to do!\n`); | |
| } else { | |
| console.log(`\nDone: ${ran} regenerated, ${skipped} skipped.\n`); | |
| } | |
| } | |
| console.log('Checking generators for changes...\n'); | |
| main().catch(err => { | |
| console.error(err); | |
| process.exit(1); | |
| }); | |