Spaces:

mahrooshishaq
/

SmartfolioBackend

Sleeping

SmartfolioBackend / src /modules /scraper /normalize.ts
deploy: WWR + Working Nomads sources + Asia/Gulf geo reachability
76a6493 4 months ago
28.3 kB
// Pure normalization functions — run ONCE at scrape/ingestion time,
// so the read path (jobs/courses pages) serves already-clean rows from the DB.

/**
 * Job-board marks come from Google's favicon service.
 *
 * These used to be logo.clearbit.com URLs. That host no longer resolves at all
 * (ERR_NAME_NOT_RESOLVED), so every card on the jobs page fired a failed image
 * request — dozens of console errors per page load and a broken mark on each
 * row. The favicon service is plain, always-available, and needs no key.
 */
const favicon = (domain: string) => `https://www.google.com/s2/favicons?domain=${domain}&sz=64`;

export const SOURCE_LOGOS: Record<string, string> = {
  'adzuna':        favicon('adzuna.com'),
  'rozee.pk':      favicon('rozee.pk'),
  'jsearch':       favicon('indeed.com'),
  'indeed':        favicon('indeed.com'),
  'linkedin':      favicon('linkedin.com'),
  'glassdoor':     favicon('glassdoor.com'),
  'bayt':          favicon('bayt.com'),
  'naukri':        favicon('naukri.com'),
  'monster':       favicon('monster.com'),
  'ziprecruiter':  favicon('ziprecruiter.com'),
  'jooble':        favicon('jooble.org'),
  // Free whole-board remote sources + ATS (direct-employer) boards.
  'himalayas':     favicon('himalayas.app'),
  'remotive':      favicon('remotive.com'),
  'jobicy':        favicon('jobicy.com'),
  'arbeitnow':     favicon('arbeitnow.com'),
  'remoteok':      favicon('remoteok.com'),
  'greenhouse':    favicon('greenhouse.io'),
  'lever':         favicon('lever.co'),
  'workingnomads': favicon('workingnomads.com'),
  'weworkremotely': favicon('weworkremotely.com'),
};

export const PLACEHOLDER_LOGO = 'https://ui-avatars.com/api/?background=0D8ABC&color=fff&name=';

export const PLATFORM_LOGOS: Record<string, string> = {
  'edx':      'https://www.edx.org/favicon.ico',
  'youtube':  'https://www.youtube.com/favicon.ico',
  'coursera': 'https://www.coursera.org/favicon.ico',
  'udemy':    'https://www.udemy.com/favicon.ico',
};

export const PLATFORM_THUMBNAILS: Record<string, string> = {
  'edx':      'https://www.edx.org/images/logos/edx-logo-elm.svg',
  'youtube':  'https://www.youtube.com/img/desktop/yt_1200.png',
  'coursera': 'https://d3njjcbhbojbot.cloudfront.net/api/utilities/v1/imageproxy/https://coursera.s3.amazonaws.com/media/coursera-logo-square.png',
  'udemy':    'https://www.udemy.com/staticx/udemy/images/v7/logo-udemy.svg',
};

const COURSE_PLACEHOLDER_LOGO = 'https://ui-avatars.com/api/?background=6366f1&color=fff&name=';

export interface NormalizedJob {
  title: string;
  company: string;
  companyLogo: string;
  location: string;
  country: string;
  salaryMin: string;
  salaryMax: string;
  /**
   * Numeric mirrors of the salary strings, or null when the board didn't say.
   * The strings stay authoritative for display ("Not specified", currency prose
   * from some boards); these exist so salary filtering can happen in SQL instead
   * of pulling every row into Node to `parseFloat` it.
   */
  salaryMinValue: number | null;
  salaryMaxValue: number | null;
  jobType: string;
  experienceLevel: string;
  category: string;
  source: string;
  sourceLogo: string;
  description: string;
  geoRestriction: string;
  applyUrl: string;
  scrapedAt: string;
  /**
   * Board-reported posting date and offer expiry, ISO strings or null when the
   * board didn't say. `expiresAt` is an explicit death signal we honour without
   * fetching anything (Phase 0 liveness); currently only JSearch provides it.
   */
  postedAt: string | null;
  expiresAt: string | null;
}

export interface NormalizedCourse {
  title: string;
  instructor: string;
  platform: string;
  platformLogo: string;
  category: string;
  level: string;
  duration: string;
  price: string;
  rating: string;
  language: string;
  description: string;
  courseUrl: string;
  thumbnail: string;
  scrapedAt: string;
  source: string;
  // Structured stats (YouTube playlists; 0/'' when the platform doesn't provide them)
  totalVideos: number;
  totalDurationMinutes: number;
  avgVideoMinutes: number;
  viewCount: number;
  likeCount: number;
  lastUpdatedAt: string;
}

// ─── Jobs ─────────────────────────────────────────────────────────────────────

/**
 * Pull a number out of whatever the board reported. Boards send "85000",
 * "£45,000", "Not specified", or nothing at all.
 *
 * Only currency symbols, separators and whitespace are stripped, and what
 * remains must be a number END TO END. Stripping every non-digit instead looks
 * tempting but silently fabricates figures: "$30 - $130/hour" collapses to
 * 30130 and "USD 50k" to 50, both of which then satisfy salary filters they
 * have no business matching. A salary we cannot read is unknown, and unknown
 * has to mean null — a wrong number is worse than no number, because the filter
 * trusts it.
 */
function parseSalaryValue(raw: unknown): number | null {
  if (raw === null || raw === undefined) return null;
  const cleaned = String(raw).replace(/[\s,$£€]/g, '');
  if (!/^\d+(\.\d+)?$/.test(cleaned)) return null;
  const value = parseFloat(cleaned);
  return Number.isFinite(value) ? value : null;
}

export function normalizeJob(raw: any, source: string): NormalizedJob | null {
  const applyUrl = raw.apply_url;
  if (!applyUrl || !String(applyUrl).startsWith('http')) return null;

  const company = raw.company || 'Not specified';
  const resolvedSource = (raw.source === 'jsearch' || source === 'jsearch')
    ? extractJsearchSource(applyUrl)
    : (raw.source || source);

  return {
    title:           raw.title || 'Not specified',
    company,
    companyLogo:     getCompanyLogo(company),
    location:        raw.location || raw.city || 'Not specified',
    country:         normalizeCountry(raw.country, raw.location),
    salaryMin:       String(raw.salary_min ?? raw.salary ?? 'Not specified'),
    salaryMax:       String(raw.salary_max ?? 'Not specified'),
    salaryMinValue:  parseSalaryValue(raw.salary_min ?? raw.salary),
    salaryMaxValue:  parseSalaryValue(raw.salary_max),
    jobType:         resolveJobType(raw),
    experienceLevel: resolveExperienceLevel(raw, resolvedSource),
    geoRestriction:  classifyGeoRestriction(raw),
    category:        normalizeCategory(raw.category, raw.title),
    source:          resolvedSource,
    sourceLogo:      SOURCE_LOGOS[resolvedSource] || PLACEHOLDER_LOGO + resolvedSource,
    description:     cleanDescription(raw.description),
    applyUrl:        String(applyUrl),
    scrapedAt:       raw.scraped_at || new Date().toISOString().split('T')[0],
    postedAt:        parseIsoOrNull(raw.posted_at),
    expiresAt:       parseIsoOrNull(raw.expires_at),
  };
}

/**
 * Parse a board-supplied date into an ISO string, or null if absent/garbage.
 * Accepts ISO datetimes ("2026-07-25T12:00:00Z"), unix seconds, and plain
 * dates. A value we can't trust becomes null rather than a fabricated date —
 * `expiresAt` gates whether a job is shown, so a wrong date would hide (or keep)
 * the wrong jobs.
 */
function parseIsoOrNull(raw: unknown): string | null {
  if (raw === null || raw === undefined || raw === '') return null;
  if (typeof raw === 'number' || /^\d{9,13}$/.test(String(raw))) {
    const n = Number(raw);
    // Heuristic: 10-digit = seconds, 13-digit = ms.
    const ms = String(raw).length <= 10 ? n * 1000 : n;
    const d = new Date(ms);
    return Number.isNaN(d.getTime()) ? null : d.toISOString();
  }
  const d = new Date(String(raw));
  return Number.isNaN(d.getTime()) ? null : d.toISOString();
}

function cleanDescription(raw: any): string {
  if (!raw || typeof raw !== 'string') return '';
  return raw
    .replace(/<[^>]+>/g, ' ')       // strip any HTML tags
    .replace(/&amp;/g, '&').replace(/&nbsp;/g, ' ')
    .replace(/&lt;/g, '<').replace(/&gt;/g, '>')
    .replace(/[ \t]+/g, ' ')
    .trim()
    .slice(0, 5000);
}

function extractJsearchSource(applyUrl: string): string {
  if (!applyUrl) return 'jsearch';
  const url = applyUrl.toLowerCase();
  if (url.includes('linkedin.com'))  return 'linkedin';
  if (url.includes('glassdoor.com')) return 'glassdoor';
  if (url.includes('indeed.com'))    return 'indeed';
  if (url.includes('bayt.com'))      return 'bayt';
  if (url.includes('rozee.pk'))      return 'rozee.pk';
  if (url.includes('naukri.com'))    return 'naukri';
  if (url.includes('monster.com'))   return 'monster';
  if (url.includes('ziprecruiter'))  return 'ziprecruiter';
  if (url.includes('jooble'))        return 'jooble';
  if (url.includes('jobgether'))     return 'jobgether';
  if (url.includes('jobrapido'))     return 'jobrapido';
  if (url.includes('expertini'))     return 'expertini';
  return 'jsearch';
}

function normalizeCountry(country: string, location: string): string {
  if (country && country.length > 2) return country;
  const codeMap: Record<string, string> = {
    'PK': 'Pakistan',       'IN': 'India',         'US': 'United States',
    'GB': 'United Kingdom', 'AU': 'Australia',     'CA': 'Canada',
    'DE': 'Germany',        'SG': 'Singapore',     'AE': 'United Arab Emirates',
  };
  const code = (country || '').toUpperCase();
  if (codeMap[code]) return codeMap[code];
  return inferCountry(location);
}

function inferCountry(location: string): string {
  if (!location) return 'Not specified';
  const l = location.toLowerCase();
  if (l.includes('pakistan') || l.includes('lahore') || l.includes('karachi') ||
      l.includes('islamabad') || l.includes('rawalpindi') || l.includes('faisalabad')) return 'Pakistan';
  if (l.includes('united states') || l.includes(', us') || l.includes(', ny') ||
      l.includes(', ca') || l.includes(', tx')) return 'United States';
  if (l.includes('united kingdom') || l.includes(', uk') || l.includes('london')) return 'United Kingdom';
  if (l.includes('australia') || l.includes('sydney') || l.includes('melbourne')) return 'Australia';
  if (l.includes('canada') || l.includes('toronto') || l.includes('vancouver')) return 'Canada';
  if (l.includes('india') || l.includes('mumbai') || l.includes('bangalore')) return 'India';
  if (l.includes('germany') || l.includes('berlin') || l.includes('munich')) return 'Germany';
  return 'Not specified';
}

// Level inference for EVERY source: declared value first, then title keywords,
// then years-of-experience mentions in the description. Adzuna never declares a
// level, so without this all its jobs were unfilterable "Not specified".
// Sources whose declared experience level is reliable enough to trust directly.
// himalayas/jobicy expose a real seniority field (mapped to our vocabulary in the
// scraper); jsearch et al. slap "Entry level" on everything, so theirs is ignored
// in favour of title/description inference below.
const TRUSTED_LEVEL_SOURCES = new Set(['adzuna', 'rozee.pk', 'himalayas', 'jobicy']);

function resolveExperienceLevel(raw: any, source: string): string {
  const declared = raw.experience_level;
  if (!TRUSTED_LEVEL_SOURCES.has(source)) {
    // jsearch et al. often lie ("Entry level" on everything) — ignore declared
  } else if (declared && declared !== 'Not specified') {
    return declared;
  }

  const t = String(raw.title || '').toLowerCase();
  if (t.match(/\bjunior\b|\bentry\b|graduate|\bintern(ship)?\b|trainee/)) return 'Entry Level';
  if (t.match(/\bsenior\b|\bsr\.?\b|lead|principal|staff/))               return 'Senior';
  if (t.match(/\bmanager\b|director|vp |head of/))                        return 'Management';
  if (t.match(/\bmid\b|intermediate|associate/))                          return 'Mid Level';

  // Fall back to "N years experience" statements in the description
  const desc = String(raw.description || '').slice(0, 2000).toLowerCase();
  const yearsMatch = desc.match(/(\d{1,2})\s*(?:-\s*\d{1,2})?\s*\+?\s*(?:years?|yrs?)(?:\s+of)?\s+(?:\w+\s+){0,3}experience/);
  if (yearsMatch) {
    const years = parseInt(yearsMatch[1], 10);
    if (years <= 1) return 'Entry Level';
    if (years <= 4) return 'Mid Level';
    return 'Senior';
  }
  if (desc.match(/\bentry[- ]level\b|no experience (necessary|required)|fresh graduate/)) return 'Entry Level';

  return 'Not specified';
}

// Remote jobs often quietly require living in a specific country or holding
// citizenship/work authorization — classify that so users can filter out jobs
// they can't actually take. Priority: citizenship signals > explicit in-country
// wording > explicit worldwide wording > structural inference (a remote job
// advertised for one specific country is normally restricted to it).
function classifyGeoRestriction(raw: any): string {
  const location = String(raw.location || raw.city || '').toLowerCase();
  const desc = String(raw.description || '').slice(0, 4000).toLowerCase();
  // The title carries the restriction surprisingly often ("Senior DevOps
  // Engineer (US REMOTE)") and used to be ignored entirely.
  const title = String(raw.title || '').toLowerCase();
  const text = `${title} ${location} ${desc}`;
  const isRemote = resolveJobType(raw) === 'Remote';

  // Punctuation-tolerant: postings write "Visa Sponsorship: Not Available" as
  // often as "no visa sponsorship", and the colon form used to slip through and
  // get labelled 'Anywhere'.
  if (/\b(us|u\.s\.|uk|eu|canadian|australian)? ?citizen(ship)?\b.{0,40}(required|only|must)|security clearance|green card|must (be (legally )?authori[sz]ed|have (the )?(legal )?right) to work|authori[sz]ed to work\b|eligible to work in|\busc\b[^.]{0,20}\bgc\b|work authori[sz]ation required|work permit required|(no|without) (visa )?sponsorship|sponsorship\W{0,4}(is\W{0,3})?not\W{0,4}(available|offered|provided|supported)|unable to (provide|offer) sponsorship|no c2c/.test(text)) {
    return 'Citizenship/visa required';
  }

  // US payroll classifications. A role offered on W2 or 1099 is administered
  // under US tax law and in practice needs US work authorization, even when the
  // posting never mentions visas — this was the single biggest source of
  // wrongly-'Anywhere' jobs.
  if (/\bw-?2\b|\b1099\b|\bc2c\b|corp[- ]to[- ]corp/.test(text)) {
    return 'Citizenship/visa required';
  }

  // "Remote (United States)" / "Remote - US" / "(US REMOTE)" bind a remote role
  // to one country without the "must reside in" phrasing the next test needs.
  if (/remote\s*[\(\[\-–—:,]\s*(the )?(usa?|u\.s\.|united states|uk|united kingdom|canada|australia|india|pakistan|germany|netherlands|eu|europe)\b|\b(usa?|uk|eu)[\s\-]remote\b|remote[\s\-](usa?|uk|eu)\b/.test(text)) {
    return 'Country-restricted';
  }

  // "Remote (EST/CST only)" — a US-timezone requirement is a residency
  // requirement in practice, however the posting phrases it.
  // Either an explicit "only", or a timezone qualifying the word remote —
  // "Remote (EST/CST)" is a residency requirement without ever saying "only".
  if (/\b(est|cst|pst|mst|edt|cdt|pdt)\b[^.]{0,30}\bonly\b|\bonly\b[^.]{0,30}\b(est|cst|pst|mst)\b|must (work|be available).{0,30}\b(est|cst|pst|mst)\b|remote\s*[\(\[\-–—:,][^)\].]{0,25}\b(est|cst|pst|mst|edt|cdt|pdt)\b/.test(text)) {
    return 'Country-restricted';
  }

  if (/remote.{0,25}\b(in|within|from|across)\b.{0,35}\b(usa?|united states|uk|united kingdom|europe|eu|canada|australia|india|pakistan|germany|netherlands)\b|only (candidates|applicants) (in|based in|located in|residing in|from)|must (reside|be located|be based|live|currently reside) in|within the (us|uk|eu|usa)|candidates? must be (in|from|based)|residents? of (the )?(us|uk|eu|usa)|\b(us|uk|eu)[- ]based (candidates|applicants|only)|local candidates only/.test(text)) {
    return 'Country-restricted';
  }

  if (/work from anywhere|anywhere in the world|(worldwide|globally) (remote|distributed|team)|remote (worldwide|globally)|fully distributed|remote[- ]first|100% remote.{0,40}(anywhere|global)|remote \((global|worldwide|anywhere)\)|open to (all|any) (locations?|countries|regions)|open to candidates (from )?(anywhere|worldwide|globally|all countries)|international (applicants|candidates).{0,20}welcome|location:?\s*(anywhere|global|worldwide|flexible)|work from any (location|country)|(emea|apac|latam|americas).{0,15}(and|,|\/).{0,15}(emea|apac|latam|americas)/.test(text)) {
    return 'Anywhere';
  }

  // A remote posting that states no citizenship, work-authorization or
  // residency requirement is treated as open, whatever market it was found in.
  //
  // This used to downgrade any remote job with a country attached to
  // 'Country-restricted (likely)'. But the country on a scraped job records
  // which market the scrape searched, not a restriction — a country-scoped
  // scrape stamps one on every single job, so 'Anywhere' became unreachable and
  // the "Work from anywhere" filter could never match anything.
  //
  // "US job + work from anywhere" is a real and common combination: the role is
  // hired for the US market but done from abroad with no authorization needed.
  // The checks above already catch the postings that do state a requirement, so
  // reaching this point means nothing was asked for.
  if (isRemote) return 'Anywhere';

  return 'Not specified';
}

function normalizeCategory(raw: string, title: string): string {
  const JUNK_CATEGORIES = [
    'linkedin', 'indeed', 'glassdoor', 'bayt', 'rozee', 'workable',
    'careers', 'jobs', 'smartrecruiters', 'jobgether', 'bebee', 'startup',
    'expertini', 'jobrapido', 'yulys', 'jaabz', 'manatal', 'spglobal',
    's&p', 'cbre', 'pepsico', 'acca', 'wizbii', 'zones', 'terra',
    'unjobnet', 'jobleads', 'crossover', 'kaispe', 'getpakjob', 'afridi',
    'webnet', 'connectx', 'sperton', 'estm', 'oraclecloud', 'ace money',
    'jazzkr', 'jazzhr', 'leading digital',
  ];

  const CATEGORY_MAP: Record<string, string> = {
    'software engineer': 'Technology',           'senior software engineer': 'Technology',
    'lead software engineer': 'Technology',      'web developer': 'Technology',
    'typescript developer': 'Technology',        'devops engineer': 'Technology',
    'cloud engineer': 'Technology',              'mobile developer': 'Technology',
    'frontend developer': 'Technology',          'backend developer': 'Technology',
    'full stack developer': 'Technology',        'database administrator': 'Technology',
    'blockchain developer': 'Technology',        'data scientist': 'Data & AI',
    'machine learning engineer': 'Data & AI',    'ai engineer': 'Data & AI',
    'ai/ml engineer': 'Data & AI',               'data analyst': 'Data & AI',
    'cybersecurity analyst': 'Cybersecurity',    'fintech software engineer': 'Finance & Accounting',
    'senior fintech developer': 'Finance & Accounting', 'finance tech lead': 'Finance & Accounting',
    'finance software engineer': 'Finance & Accounting', 'ai engineer finance': 'Finance & Accounting',
    'finance manager': 'Finance & Accounting',   'accountant': 'Finance & Accounting',
    'investment analyst': 'Finance & Accounting','financial analyst': 'Finance & Accounting',
    'business analyst': 'Product & Project Management', 'product manager': 'Product & Project Management',
    'project manager': 'Product & Project Management',  'supply chain manager': 'Operations',
    'operations manager': 'Operations',          'business development': 'Sales & Business Development',
    'sales manager': 'Sales & Business Development',    'account manager': 'Sales & Business Development',
    'digital marketing': 'Marketing',            'content marketing': 'Marketing',
    'seo specialist': 'Marketing',               'social media manager': 'Marketing',
    'marketing manager': 'Marketing',            'graphic designer': 'Design & Creative',
    'ui ux designer': 'Design & Creative',       'product designer': 'Design & Creative',
    'human resources manager': 'Human Resources','recruiter': 'Human Resources',
    'talent acquisition': 'Human Resources',     'customer service': 'Customer Service',
    'technical support': 'Customer Service',
  };

  if (!raw || raw === 'Not specified') return inferJobCategoryFromTitle(title);

  const rawLower = raw.toLowerCase().trim();
  if (CATEGORY_MAP[rawLower]) return CATEGORY_MAP[rawLower];

  const isJunk = JUNK_CATEGORIES.some(j => rawLower.includes(j));
  if (isJunk) return inferJobCategoryFromTitle(title);

  const looksLikeCompany =
    /\.(com|pk|org|net|io)$/i.test(raw) ||
    (/^[A-Z][a-z]+ [A-Z][a-z]+/.test(raw) &&
      raw.split(' ').length <= 3 &&
      !raw.toLowerCase().match(/manager|engineer|analyst|developer|designer|specialist|officer/));

  if (looksLikeCompany) return inferJobCategoryFromTitle(title);
  return raw;
}

function inferJobCategoryFromTitle(title: string): string {
  if (!title) return 'Not specified';
  const t = title.toLowerCase();
  if (t.match(/software|developer|engineer|frontend|backend|fullstack|full stack|devops|cloud|mobile|flutter|react|angular|vue|node|python|java|\.net/)) return 'Technology';
  if (t.match(/data\s*(scientist|analyst|engineer)|machine learning|ai |artificial|nlp|deep learning/)) return 'Data & AI';
  if (t.match(/cyber|security|infosec|penetration/)) return 'Cybersecurity';
  if (t.match(/product manager|project manager|scrum|agile|program manager/)) return 'Product & Project Management';
  if (t.match(/marketing|seo|content|social media|brand|growth/)) return 'Marketing';
  if (t.match(/sales|account manager|business development|bd /)) return 'Sales & Business Development';
  if (t.match(/finance|financial|accountant|accounting|audit|tax|investment|analyst/)) return 'Finance & Accounting';
  if (t.match(/fintech|fin tech/)) return 'Finance & Accounting';
  if (t.match(/hr |human resource|recruiter|talent|people ops/)) return 'Human Resources';
  if (t.match(/design|ui|ux|graphic|creative|figma|illustrat/)) return 'Design & Creative';
  if (t.match(/customer|support|service|helpdesk|success/)) return 'Customer Service';
  if (t.match(/operations|supply chain|logistics|procurement/)) return 'Operations';
  return 'General';
}

// Sources often misreport work mode (JSearch's is_remote flag misses jobs whose
// location or description plainly says "Remote"), so infer from text when the
// declared type carries no work-mode signal.
function resolveJobType(raw: any): string {
  const declared = normalizeJobType(raw.job_type);
  if (declared === 'Remote' || declared === 'Hybrid' || declared === 'Onsite') return declared;

  const location = String(raw.location || raw.city || '').toLowerCase();
  const title = String(raw.title || '').toLowerCase();
  const desc = String(raw.description || '').slice(0, 1500).toLowerCase();

  if (location.includes('remote') || title.includes('remote') ||
      /\b(fully remote|100% remote|work from home|remote position|remote job|location:\s*remote|job type:\s*remote)\b/.test(desc)) {
    return 'Remote';
  }
  if (location.includes('hybrid') || title.includes('hybrid') || /\bhybrid\b/.test(desc)) {
    return 'Hybrid';
  }
  return declared;
}

function normalizeJobType(raw: string): string {
  if (!raw || raw === 'Not specified') return 'Not specified';
  const val = raw.toLowerCase().trim();
  if (val.includes('full'))      return 'Full Time';
  if (val.includes('part'))      return 'Part Time';
  if (val.includes('remote'))    return 'Remote';
  if (val.includes('hybrid'))    return 'Hybrid';
  if (val.includes('onsite') || val.includes('on-site') || val.includes('on site')) return 'Onsite';
  if (val.includes('contract'))  return 'Contract';
  if (val.includes('intern'))    return 'Internship';
  if (val.includes('permanent')) return 'Full Time';
  return raw;
}

/**
 * Company marks are generated initials, not fetched logos.
 *
 * This used to guess a domain from the company name and ask clearbit for its
 * logo. Two problems: that host is dead, and the guess was wrong far more often
 * than it was right — "Hire Feed" is not hirefeed.com. Initials on a tinted
 * square always render, never 404, and never mislabel a company with someone
 * else's brand.
 */
function getCompanyLogo(company: string): string {
  const name = !company || company === 'Not specified' ? '?' : company;
  return PLACEHOLDER_LOGO + encodeURIComponent(name);
}

// ─── Courses ──────────────────────────────────────────────────────────────────

export function normalizeCourse(raw: any): NormalizedCourse | null {
  const courseUrl = raw.course_url;
  if (!courseUrl || !String(courseUrl).startsWith('http')) return null;

  const source = (raw.source || 'unknown').toLowerCase();
  const platform = raw.platform || inferPlatform(source);

  return {
    title:        raw.title       || 'Not specified',
    instructor:   raw.instructor  || 'Not specified',
    platform,
    platformLogo: PLATFORM_LOGOS[source] || COURSE_PLACEHOLDER_LOGO + platform,
    category:     normalizeCourseCategory(raw.category, raw.title),
    level:        normalizeLevel(raw.level),
    duration:     raw.duration    || 'Not specified',
    price:        raw.price       || 'Not specified',
    rating:       String(raw.rating ?? 'Not specified'),
    language:     raw.language    || 'English',
    description:  raw.description || 'Not specified',
    courseUrl:    String(courseUrl),
    thumbnail:    resolveThumbnail(raw.thumbnail, source),
    scrapedAt:    raw.scraped_at  || new Date().toISOString().split('T')[0],
    source,
    totalVideos:          toInt(raw.total_videos),
    totalDurationMinutes: toInt(raw.total_duration_minutes),
    avgVideoMinutes:      toInt(raw.avg_video_minutes),
    viewCount:            toInt(raw.view_count),
    likeCount:            toInt(raw.like_count),
    lastUpdatedAt:        typeof raw.last_updated_at === 'string' ? raw.last_updated_at : '',
  };
}

function toInt(v: any): number {
  const n = Number(v);
  return Number.isFinite(n) && n > 0 ? Math.floor(n) : 0;
}

function resolveThumbnail(thumbnail: string, source: string): string {
  if (thumbnail && thumbnail.startsWith('http')) return thumbnail;
  return PLATFORM_THUMBNAILS[source] || COURSE_PLACEHOLDER_LOGO + source;
}

function inferPlatform(source: string): string {
  const map: Record<string, string> = {
    'edx': 'edX', 'youtube': 'YouTube', 'coursera': 'Coursera', 'udemy': 'Udemy',
  };
  return map[source] || source;
}

function normalizeLevel(raw: string): string {
  if (!raw || raw === 'Not specified') return 'Not specified';
  const val = raw.toLowerCase();
  if (val.includes('beginner') || val.includes('basic') || val.includes('introduct')) return 'Beginner';
  if (val.includes('intermediate') || val.includes('mid'))  return 'Intermediate';
  if (val.includes('advanced') || val.includes('expert'))   return 'Advanced';
  if (val.includes('all'))                                   return 'All Levels';
  return raw;
}

function normalizeCourseCategory(raw: string, title: string): string {
  if (!raw || raw === 'Not specified' || raw === 'General') {
    return inferCourseCategoryFromTitle(title);
  }
  return raw;
}

function inferCourseCategoryFromTitle(title: string): string {
  if (!title) return 'General';
  const t = title.toLowerCase();
  if (t.match(/typescript|javascript|react|node|python|java|\.net|programming|software|developer|web dev|backend|frontend|fullstack|devops/)) return 'Technology';
  if (t.match(/machine learning|ai |artificial|deep learning|nlp|data science|neural/)) return 'Data & AI';
  if (t.match(/fintech|fin tech|financial technology|blockchain|crypto|defi/)) return 'Finance & Technology';
  if (t.match(/finance|investment|accounting|trading|banking|economics/)) return 'Finance';
  if (t.match(/cyber|security|ethical hacking|penetration/)) return 'Cybersecurity';
  if (t.match(/product|project management|agile|scrum/)) return 'Product Management';
  if (t.match(/marketing|seo|growth|branding/)) return 'Marketing';
  if (t.match(/design|ui|ux|figma|graphic/)) return 'Design';
  if (t.match(/sql|database|postgresql|mongodb/)) return 'Database';
  if (t.match(/cloud|aws|azure|gcp|kubernetes|docker/)) return 'Cloud & DevOps';
  return 'General';
}