Spaces:
Sleeping
Sleeping
File size: 1,396 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | import { Document } from "../../../controllers/v2/types";
import { Meta } from "..";
import { extractAttributes } from "../lib/extractAttributes";
import { hasFormatOfType } from "../../../lib/format-utils";
/**
* Transformer to extract attributes from HTML using the attributes format
*/
export async function performAttributes(
meta: Meta,
document: Document,
): Promise<Document> {
const attributesFormat = hasFormatOfType(meta.options.formats, "attributes");
if (!attributesFormat) {
return document;
}
if (document.html === undefined) {
throw new Error(
"html is undefined -- this transformer is being called out of order",
);
}
if (attributesFormat.selectors && attributesFormat.selectors.length > 0) {
try {
const attributes = await extractAttributes(
document.html,
attributesFormat.selectors,
);
if (attributes.length > 0) {
document.attributes = attributes;
meta.logger.debug("Extracted attributes", {
count: attributes.length,
attributes: attributes.map(d => ({
selector: d.selector,
attribute: d.attribute,
valuesCount: d.values.length,
})),
});
}
} catch (error) {
meta.logger.error("Failed to extract attributes", {
error,
url: meta.url,
});
}
}
return document;
}
|