Spaces:
Sleeping
Sleeping
File size: 4,213 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 | /**
* HTTP Client for HTML to Markdown conversion service
*
* This client communicates with the Go-based HTML to Markdown microservice
* to avoid blocking Node.js event loop with heavy conversions.
*/
import axios, { AxiosInstance, AxiosError } from "axios";
import { config } from "../config";
import { logger } from "./logger";
import type { Logger } from "winston";
import * as Sentry from "@sentry/node";
interface ConvertRequest {
html: string;
}
interface ConvertResponse {
markdown: string;
success: boolean;
}
interface ErrorResponse {
error: string;
details?: string;
success: boolean;
}
/**
* Convert HTML to Markdown using direct axios call
* @param html HTML string to convert
* @param context Optional context with logger and requestId
* @returns Markdown string
* @throws Error if conversion fails
*/
export async function convertHTMLToMarkdownWithHttpService(
html: string,
context?: {
logger?: Logger;
requestId?: string;
zeroDataRetention?: boolean;
},
): Promise<string> {
if (!html || html.trim() === "") {
return "";
}
const contextLogger = context?.logger || logger;
const requestId = context?.requestId;
const zeroDataRetention = context?.zeroDataRetention === true;
const url = config.HTML_TO_MARKDOWN_SERVICE_URL;
const startTime = Date.now();
try {
const request: ConvertRequest = { html };
const headers: Record<string, string> = {
"Content-Type": "application/json",
};
// Add request ID header if available, but never for ZDR — it would
// let the downstream service correlate logs back to a customer scrape.
if (requestId && !zeroDataRetention) {
headers["X-Request-ID"] = requestId;
}
if (zeroDataRetention) {
headers["X-Zero-Data-Retention"] = "true";
}
const response = await axios.post<ConvertResponse>(
`${url}/convert`,
request,
{
timeout: 60_000,
headers,
},
);
const duration = Date.now() - startTime;
if (!response.data.success) {
throw new Error("Conversion was not successful");
}
if (!zeroDataRetention) {
contextLogger.debug("HTML to Markdown conversion successful", {
duration_ms: duration,
input_size: html.length,
output_size: response.data.markdown.length,
...(requestId ? { request_id: requestId } : {}),
});
}
return response.data.markdown;
} catch (error) {
const duration = Date.now() - startTime;
if (axios.isAxiosError(error)) {
const axiosError = error as AxiosError<ErrorResponse>;
const errorMessage =
axiosError.response?.data?.error || axiosError.message;
const errorDetails = axiosError.response?.data?.details;
const statusCode = axiosError.response?.status;
contextLogger.error("HTML to Markdown conversion failed", {
error: errorMessage,
details: errorDetails,
statusCode,
duration_ms: duration,
serviceUrl: url,
});
// Capture in Sentry with additional context (omit identifying fields
// and content sizes when ZDR so nothing is retained in Sentry either).
Sentry.captureException(error, {
tags: {
service: "html-to-markdown",
status_code: statusCode,
...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),
},
extra: {
serviceUrl: url,
errorMessage,
errorDetails,
...(zeroDataRetention ? {} : { inputSize: html.length }),
},
});
// Include details in error message if available
const fullErrorMessage = errorDetails
? `HTML to Markdown conversion failed: ${errorMessage} - ${errorDetails}`
: `HTML to Markdown conversion failed: ${errorMessage}`;
throw new Error(fullErrorMessage);
} else {
contextLogger.error(
"Unexpected error during HTML to Markdown conversion",
{
error,
},
);
Sentry.captureException(error, {
tags: {
...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),
},
});
throw error;
}
}
}
|