| #!/usr/bin/env node |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| const path = require('path') |
| const fs = require('fs') |
| const https = require('https') |
|
|
| const EmojiDbLib = require('emoji-db') |
| const emojiDb = new EmojiDbLib({ useDefaultDb: true }) |
|
|
| const EMOJI_DIR = path.resolve(__dirname, '../assets/emoji/') |
|
|
| const CDN_VERSION = '16.0.0' |
| const CDN_SIZE = 64 |
| const CONCURRENCY = 50 |
|
|
| const SUPPORTED_BRANDS = { |
| apple: { |
| pkg: 'emoji-datasource-apple', |
| folder: 'apple' |
| }, |
| google: { |
| pkg: 'emoji-datasource-google', |
| folder: 'google' |
| }, |
| twitter: { |
| pkg: 'emoji-datasource-twitter', |
| folder: 'twitter' |
| } |
| } |
|
|
| const BRAND_JSON_FILES = { |
| apple: 'emoji-apple-image.json', |
| google: 'emoji-google-image.json', |
| twitter: 'emoji-twitter-image.json', |
| joypixels: 'emoji-joypixels-image.json', |
| blob: 'emoji-blob-image.json' |
| } |
|
|
| function fetchBuffer (url) { |
| return new Promise((resolve, reject) => { |
| const req = https.get(url, (res) => { |
| if (res.statusCode !== 200) { |
| res.resume() |
| return reject(new Error(`HTTP ${res.statusCode} for ${url}`)) |
| } |
| const chunks = [] |
| res.on('data', (chunk) => chunks.push(chunk)) |
| res.on('end', () => resolve(Buffer.concat(chunks))) |
| res.on('error', reject) |
| }) |
| req.on('error', reject) |
| req.setTimeout(10000, () => { |
| req.destroy() |
| reject(new Error(`Timeout for ${url}`)) |
| }) |
| }) |
| } |
|
|
| function getCdnUrl (brand, emojiCode) { |
| const config = SUPPORTED_BRANDS[brand] |
| return `https://cdn.jsdelivr.net/npm/${config.pkg}@${CDN_VERSION}/img/${config.folder}/${CDN_SIZE}/${emojiCode}.png` |
| } |
|
|
| async function runPool (tasks, concurrency) { |
| const results = [] |
| let index = 0 |
|
|
| async function worker () { |
| while (index < tasks.length) { |
| const i = index++ |
| results[i] = await tasks[i]().catch(() => null) |
| } |
| } |
|
|
| const workers = Array.from({ length: Math.min(concurrency, tasks.length) }, () => worker()) |
| await Promise.all(workers) |
| return results |
| } |
|
|
| async function updateBrand (brand) { |
| const config = SUPPORTED_BRANDS[brand] |
| if (!config) { |
| console.log(` Skipping ${brand} — not supported by emoji-datasource (keeping existing JSON)`) |
| return |
| } |
|
|
| const jsonFile = path.join(EMOJI_DIR, BRAND_JSON_FILES[brand]) |
|
|
| |
| let existing = {} |
| try { |
| if (fs.existsSync(jsonFile)) { |
| existing = JSON.parse(fs.readFileSync(jsonFile, 'utf8')) |
| } |
| } catch (err) { |
| console.warn(` Warning: could not read existing ${jsonFile}: ${err.message}`) |
| } |
|
|
| const dbData = emojiDb.dbData |
| const codes = Object.keys(dbData).filter(k => !dbData[k].qualified) |
|
|
| |
| const missing = codes.filter(code => !existing[code]) |
| const total = codes.length |
| const alreadyHave = total - missing.length |
|
|
| console.log(` ${brand}: ${total} emoji total, ${alreadyHave} cached, ${missing.length} to download`) |
|
|
| if (missing.length === 0) { |
| console.log(` ${brand}: up to date`) |
| return |
| } |
|
|
| let downloaded = 0 |
| let failed = 0 |
|
|
| const tasks = missing.map(code => async () => { |
| try { |
| const url = getCdnUrl(brand, code) |
| const buffer = await fetchBuffer(url) |
| const base64 = buffer.toString('base64') |
|
|
| if (base64.length > 0) { |
| downloaded++ |
| if (downloaded % 100 === 0) process.stdout.write(` ${brand}: ${downloaded}/${missing.length}\r`) |
| return { code, base64 } |
| } |
| } catch (err) { |
| failed++ |
| return null |
| } |
| }) |
|
|
| const results = await runPool(tasks, CONCURRENCY) |
|
|
| for (const result of results) { |
| if (result) existing[result.code] = result.base64 |
| } |
|
|
| console.log(` ${brand}: downloaded ${downloaded}, failed ${failed}, total ${Object.keys(existing).length}`) |
|
|
| |
| const tmpFile = jsonFile + '.tmp' |
| fs.writeFileSync(tmpFile, JSON.stringify(existing)) |
| fs.renameSync(tmpFile, jsonFile) |
|
|
| console.log(` ${brand}: saved to ${path.basename(jsonFile)}`) |
| } |
|
|
| async function main () { |
| const args = process.argv.slice(2) |
| const brands = args.length > 0 ? args : Object.keys(BRAND_JSON_FILES) |
|
|
| console.log('Emoji updater — CDN: emoji-datasource@' + CDN_VERSION + ' via jsdelivr') |
| console.log('Brands to update:', brands.join(', ')) |
| console.log() |
|
|
| for (const brand of brands) { |
| if (!BRAND_JSON_FILES[brand]) { |
| console.error(`Unknown brand: ${brand}`) |
| continue |
| } |
| await updateBrand(brand) |
| } |
|
|
| console.log('\nDone.') |
| } |
|
|
| main().catch(err => { |
| console.error('Fatal error:', err) |
| process.exit(1) |
| }) |
|
|
