File size: 4,830 Bytes
097eaf0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
/**
 * Transformer: Send Document to Search Index
 *
 * Integrates with the existing scraper transformer stack.
 * Queues documents for real-time search indexing.
 *
 * Sampling: Controlled via SEARCH_INDEX_SAMPLE_RATE (0.0-1.0)
 * - 0.1 = 10% of documents indexed (recommended for initial rollout)
 * - 1.0 = 100% of documents indexed (full production)
 */

import { Document } from "../../../controllers/v1/types";
import { config } from "../../../config";
import { indexDocumentIfEnabled } from "../../../lib/search-index-client";
import { logger as _logger } from "../../../lib/logger";
import { Meta } from "..";

/**
 * Check if document should be indexed for search
 */
function shouldIndexForSearch(meta: Meta, document: Document): boolean {
  if (meta.internalOptions.isParse) {
    return false;
  }

  if (meta.internalOptions.zeroDataRetention) {
    return false;
  }

  // Lockdown must not forward the target URL to any external service.
  if (meta.options.lockdown) {
    return false;
  }

  const statusCode = document.metadata.statusCode;
  if (statusCode < 200 || statusCode >= 300) {
    return false;
  }

  // Check if markdown content exists and is substantial
  const markdown = document.markdown ?? "";
  if (markdown.length < 200) {
    return false;
  }

  // Don't index if has auth headers (private content)
  if (
    meta.options.headers &&
    (meta.options.headers["Authorization"] || meta.options.headers["Cookie"])
  ) {
    return false;
  }

  // Don't index PDFs without parsing
  const isPdf = document.metadata.contentType?.includes("pdf");
  if (isPdf && !document.markdown) {
    return false;
  }

  return true;
}

/**
 * Determine if this document should be sampled for indexing
 */
function shouldSampleDocument(): boolean {
  // Get sample rate from environment (default 10% for safe rollout)
  const sampleRate = config.SEARCH_INDEX_SAMPLE_RATE;

  // Validate sample rate
  if (isNaN(sampleRate) || sampleRate < 0 || sampleRate > 1) {
    _logger.warn("Invalid SEARCH_INDEX_SAMPLE_RATE, using 0.1 (10%)", {
      value: sampleRate,
    });
    return Math.random() < 0.1;
  }

  // Sample based on rate
  return Math.random() < sampleRate;
}

/**
 * Transformer: Send document to search index
 */
export async function sendDocumentToSearchIndex(
  meta: Meta,
  document: Document,
): Promise<Document> {
  // Check if search indexing is enabled via the SEARCH_SERVICE_URL
  const searchIndexEnabled =
    config.ENABLE_SEARCH_INDEX && config.SEARCH_SERVICE_URL;

  meta.logger.debug("Sending document to search index", {
    url: meta.url,
    searchIndexEnabled,
  });

  if (!searchIndexEnabled) {
    return document;
  }

  // Apply sampling (canary rollout)
  if (!shouldSampleDocument()) {
    meta.logger.debug("Document not sampled for search indexing", {
      url: meta.url,
      sampleRate: config.SEARCH_INDEX_SAMPLE_RATE,
    });
    return document;
  }

  // Check if document should be indexed
  if (!shouldIndexForSearch(meta, document)) {
    meta.logger.debug("Document not suitable for search index", {
      url: meta.url,
      statusCode: document.metadata.statusCode,
      markdownLength: document.markdown?.length ?? 0,
    });
    return document;
  }

  // Get the indexId from document metadata (set by sendDocumentToIndex transformer)
  // Format as GCS path: {indexId}.json
  const gcsPath = document.metadata.indexId;

  // Remove indexId from metadata after extracting it (internal field, shouldn't be exposed to user)
  delete document.metadata.indexId;

  // Send to search service via HTTP (async, don't block scraper)
  (async () => {
    try {
      await indexDocumentIfEnabled(
        {
          url: meta.url,
          resolvedUrl:
            document.metadata.url ??
            document.metadata.sourceURL ??
            meta.rewrittenUrl ??
            meta.url,
          title:
            document.metadata.title ?? document.metadata.ogTitle ?? undefined,
          description:
            document.metadata.description ??
            document.metadata.ogDescription ??
            undefined,
          markdown: document.markdown ?? "",
          html: document.rawHtml ?? "",
          statusCode: document.metadata.statusCode,
          gcsPath: gcsPath,
          screenshotUrl: document.screenshot ?? undefined,
          language: document.metadata.language ?? "en",
          country: meta.options.location?.country ?? undefined,
          isMobile: meta.options.mobile ?? false,
        },
        meta.logger,
      );

      meta.logger.debug("Sent document to search service", {
        url: meta.url,
      });
    } catch (error) {
      meta.logger.error("Failed to send document to search service", {
        error: (error as Error).message,
        url: meta.url,
      });
    }
  })();

  return document;
}