Blame for src/modules/scraper/scraper.controller.ts

import { Controller, Post, Get, Param, Body, UseGuards, Req, Query } from '@nestjs/common';
import { ScraperService } from './scraper.service';
import { MaintenanceService } from './maintenance.service';
import { LinkVerifierService } from './link-verifier.service';
import { JobsService } from '../jobs/jobs.service';
import { CoursesService } from '../courses/courses.service';
import { ApiTags, ApiOperation, ApiBearerAuth, ApiQuery, ApiSecurity } from '@nestjs/swagger';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { AdminKeyGuard } from '../auth/admin-key.guard';
import { CustomSearchDto } from './dto/custom-search.dto';
import { UsersService } from '../users/users.service';

@ApiTags('Scraper')
@Controller('scraper')
export class ScraperController {
  constructor(
    private readonly scraperService: ScraperService,
    private readonly jobsService: JobsService,
    private readonly coursesService: CoursesService,
    private readonly maintenanceService: MaintenanceService,
    private readonly usersService: UsersService,
    private readonly linkVerifier: LinkVerifierService,
  ) {}

  // ─── POST /scraper/maintenance/expire — manually run job expiry/cleanup ─────
  @Post('maintenance/expire')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({ summary: 'Admin: run job expiry + cleanup now (normally daily 02:00 cron)' })
  async runExpiry() {
    await this.maintenanceService.expireAndCleanJobs();
    return { status: 'done' };
  }

  // ─── POST /scraper/maintenance/refresh — manually run nightly refresh ───────
  @Post('maintenance/refresh')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({ summary: 'Admin: run scheduled refresh for active users now (normally nightly 03:00 cron)' })
  async runRefresh() {
    await this.maintenanceService.refreshActiveUsers();
    return { status: 'done' };
  }

  // ─── POST /scraper/maintenance/purge-jobs — wipe the feed for a fresh start ──
  @Post('maintenance/purge-jobs')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({
    summary: 'Admin: delete ALL scraped jobs + link_health + job cache (fresh start)',
    description:
      'DESTRUCTIVE. Deletes every scraped_jobs row, all link_health verdicts, and the job-query ' +
      'cache so the next scrape re-fetches fresh. Preserves job_applications (saved tracker), courses, ' +
      'and scrape history. Requires ?confirm=DELETE to actually run.',
  })
  @ApiQuery({ name: 'confirm', required: false })
  async purgeJobs(@Query('confirm') confirm?: string) {
    if (confirm !== 'DELETE') {
      return {
        error: 'No-op. Pass ?confirm=DELETE to actually purge.',
        wouldDelete: 'ALL scraped_jobs + link_health + job cache',
        preserves: 'job_applications, courses, scrape history',
      };
    }
    return this.maintenanceService.purgeAllJobs();
  }

  // ─── POST /scraper/maintenance/verify-links ─────────────────────────────────
  @Post('maintenance/verify-links')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({
    summary: 'Admin: verify apply-link liveness across all currently-visible jobs',
    description:
      'Fetches every distinct apply URL of non-expired jobs, follows redirects, and records ' +
      'live/dead/unknown in link_health. Non-destructive. It never edits job rows; dead links ' +
      'are hidden by the feed JOIN and can be un-hidden by clearing link_health. Fail-open: only ' +
      'a positive death signal (404/410/removed-page) marks a link dead; blocked/timeout stays ' +
      'shown. Run from the deployed backend (its IP is not bot-blocked like a laptop/CI IP). ' +
      'Query: ?limit=N caps URLs checked; ?force=true re-checks even fresh ones.',
  })
  @ApiQuery({ name: 'limit', required: false, type: Number })
  @ApiQuery({ name: 'force', required: false, type: Boolean })
  async verifyLinks(@Query('limit') limit?: number, @Query('force') force?: string) {
    return this.maintenanceService.verifyExistingJobLinks({
      limit: limit ? Number(limit) : undefined,
      force: force === 'true' || force === '1',
    });
  }

  // ─── GET /scraper/maintenance/link-health — audit verdicts + evidence ───────
  @Get('maintenance/link-health')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({
    summary: 'Admin: review link-liveness verdicts with the evidence behind each',
    description:
      'The audit view for the filter. Lists link_health rows of a given status (default "dead") ' +
      'with the exact reason, HTTP status, final resolved URL, and evidence snippet, plus a ' +
      'representative job. Use it to confirm every discarded job was rightly discarded before ' +
      'trusting the pass. ?status=dead|live|unknown|pending|all, ?limit=N.',
  })
  @ApiQuery({ name: 'status', required: false, enum: ['dead', 'live', 'unknown', 'pending', 'all'] })
  @ApiQuery({ name: 'limit', required: false, type: Number })
  async linkHealth(@Query('status') status?: string, @Query('limit') limit?: number) {
    const allowed = ['dead', 'live', 'unknown', 'pending', 'all'];
    const s = (allowed.includes(status as string) ? status : 'dead') as any;
    return this.maintenanceService.listLinkHealth(s, limit ? Number(limit) : 100);
  }

  // ─── POST /scraper/maintenance/verify-url — re-check ONE url live (audit) ────
  @Post('maintenance/verify-url')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({
    summary: 'Admin: verify a single apply URL live and return the full verdict + evidence',
    description:
      'On-demand re-check of one URL (escalating to a browser on bot-hostile boards), for auditing ' +
      'a specific discard. Returns { status, finalUrl, httpStatus, reason, evidence }. Body: { "url": "..." }.',
  })
  async verifyUrl(@Body() body: { url?: string }) {
    if (!body?.url || !/^https?:\/\//i.test(body.url)) {
      return { error: 'Provide a valid http(s) url in the body: { "url": "..." }' };
    }
    const row = await this.linkVerifier.verifyOne(body.url, true);
    return {
      url: row.url, status: row.status, finalUrl: row.finalUrl,
      httpStatus: row.httpStatus, reason: row.reason, evidence: row.evidence,
      escalated: row.escalated, lastCheckedAt: row.lastCheckedAt,
    };
  }

  // ─── POST /scraper/maintenance/backfill-salaries ────────────────────────────
  @Post('maintenance/backfill-salaries')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({
    summary: 'Admin: fill numeric salary columns from the original strings (one-time)',
    description:
      'Jobs written before the numeric salary columns existed have NULL mirrors, and the ' +
      'salary filter requires NOT NULL, so filtering by salary returns nothing for them. ' +
      'Run once after deploying. Safe to repeat: it only touches rows still NULL.',
  })
  async backfillSalaries() {
    return this.maintenanceService.backfillSalaryValues();
  }

  // ─── POST /scraper/run ───────────────────────────────────────────────────────
  @Post('run')
  @UseGuards(JwtAuthGuard)
  @ApiBearerAuth()
  @ApiOperation({
    summary: 'Run personalized scraper for logged-in user',
    description: 'Reads user onboarding profile, generates targeted queries via AI, scrapes relevant jobs and courses, stores results in the database',
  })
  async runScraper(@Req() req: any) {
    const userId = req.user.id;
    return this.scraperService.runForUser(userId);
  }

  // ─── POST /scraper/search — Custom query search (async) ─────────────────────
  @Post('search')
  @UseGuards(JwtAuthGuard)
  @ApiBearerAuth()
  @ApiOperation({
    summary: 'Start a custom search (async)',
    description: 'Kicks off scraping for a user-typed query in the background and returns a jobId immediately. ' +
      'Poll GET /scraper/search/:jobId for status/result instead of blocking for 30s–5min. ' +
      'Any `filters` are pushed down into the job-board APIs, so results come back already narrowed ' +
      'rather than being filtered out of a broad pool afterwards.',
  })
  searchScraper(@Req() req: any, @Body() body: CustomSearchDto) {
    // Typing a search is deliberate intent — it keeps the nightly refresh alive
    // for this user. Not awaited; the search itself must not wait on bookkeeping.
    void this.usersService.markActive(req.user.id).catch(() => undefined);
    const job = this.scraperService.startCustomSearch(
      req.user.id, body.query || '', body.type || 'both', body.filters,
    );
    return { jobId: job.jobId, status: job.status, query: job.query, filters: job.filters };
  }

  // ─── GET /scraper/search/:jobId — poll a custom search ──────────────────────
  @Get('search/:jobId')
  @UseGuards(JwtAuthGuard)
  @ApiBearerAuth()
  @ApiOperation({ summary: 'Poll the status/result of a custom search started via POST /scraper/search' })
  getSearchStatus(@Req() req: any, @Param('jobId') jobId: string) {
    // Scoped to the caller: another user's jobId reads as not_found.
    const job = this.scraperService.getCustomSearchStatus(jobId, req.user.id);
    if (!job) return { status: 'not_found', message: 'Unknown or expired jobId. Start a new search.' };
    return job;
  }

  // ─── POST /scraper/jobs/:jobId/resolve — JIT liveness check at click time ────
  @Post('jobs/:jobId/resolve')
  @UseGuards(JwtAuthGuard)
  @ApiBearerAuth()
  @ApiOperation({
    summary: 'Verify a job apply-link at click time and return where it actually lands',
    description:
      'Checks the posting live (HTTP, escalating to a browser on bot-hostile boards), follows the ' +
      'redirect chain, and returns { status, finalUrl }. The frontend opens finalUrl for live/unknown, ' +
      'or warns when dead. Persists to link_health so the verdict also removes a dead job from the feed. ' +
      'Fail-open: only a positive death signal yields status "dead".',
  })
  async resolveJobLink(@Req() req: any, @Param('jobId') jobId: string) {
    const applyUrl = await this.jobsService.getApplyUrl(req.user.id, jobId);
    if (!applyUrl) return { status: 'not_found', finalUrl: '', applyUrl: '' };
    const health = await this.linkVerifier.verifyOne(applyUrl, true);
    return {
      status: health.status,
      finalUrl: health.finalUrl || applyUrl,
      applyUrl,
      reason: health.reason,
    };
  }

  // ─── GET /scraper/runs — scrape history for logged-in user ──────────────────
  @Get('runs')
  @UseGuards(JwtAuthGuard)
  @ApiBearerAuth()
  @ApiOperation({
    summary: 'Get scrape run history for logged-in user',
    description: 'Shows when scrapes ran, which platforms succeeded/failed, and how many items each found',
  })
  @ApiQuery({ name: 'limit', required: false, type: Number })
  async getMyRuns(@Req() req: any, @Query('limit') limit?: number) {
    return this.scraperService.getRunHistory(req.user.id, Number(limit) || 20);
  }

  // ─── POST /scraper/migrate-legacy ────────────────────────────────────────────
  @Post('migrate-legacy')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({
    summary: 'Admin: one-time import of old per-user JSON files into the database',
    description: 'Reads src/data/users/*.json and upserts all jobs/courses into the DB. Safe to run multiple times (deduped by URL).',
  })
  async migrateLegacy() {
    return this.scraperService.migrateLegacyFiles();
  }

  // ─── POST /scraper/run/:userId ───────────────────────────────────────────────
  // Admin-only: this spends Adzuna/JSearch/Groq quota, so it must never be
  // reachable without the operator key.
  @Post('run/:userId')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({ summary: 'Admin: run scraper for a specific user by ID' })
  async runScraperForUser(@Param('userId') userId: string) {
    return this.scraperService.runForUser(userId);
  }

  // ─── GET /scraper/runs/:userId ───────────────────────────────────────────────
  @Get('runs/:userId')
  @UseGuards(AdminKeyGuard)
  @ApiSecurity('admin-key')
  @ApiOperation({ summary: 'Admin: get scrape run history for a specific user' })
  async getUserRuns(@Param('userId') userId: string, @Query('limit') limit?: number) {
    return this.scraperService.getRunHistory(userId, Number(limit) || 20);
  }

  // ─── GET /scraper/results — scraped results for the logged-in user ───────────
  // Was GET /scraper/results/:userId, which let any authenticated caller read
  // any other user's jobs and courses. The id now comes from the token.
  @Get('results')
  @UseGuards(JwtAuthGuard)
  @ApiBearerAuth()
  @ApiOperation({ summary: 'Get scraped results for the logged-in user.',
    description: 'Returned fields for courses: title, instructor, platform, category, level, duration, price, rating, language, description, course_url, thumbnail, scraped_at, source. Returned fields for jobs: title, company, location, salary_min, salary_max, job_type, experience_level, category, country, source, apply_url, scraped_at' })
  async getResults(@Req() req: any) {
    const userId = req.user.id;
    const [jobs, courses] = await Promise.all([
      this.jobsService.getUserJobs(userId),
      this.coursesService.getUserCourses(userId),
    ]);

    if (jobs.length === 0 && courses.length === 0) {
      return { message: 'No results yet. Please run POST /scraper/run first.' };
    }

    return { jobs, courses };
  }
}