File size: 1,396 Bytes
097eaf0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
import { Document } from "../../../controllers/v2/types";
import { Meta } from "..";
import { extractAttributes } from "../lib/extractAttributes";
import { hasFormatOfType } from "../../../lib/format-utils";

/**
 * Transformer to extract attributes from HTML using the attributes format
 */
export async function performAttributes(
  meta: Meta,
  document: Document,
): Promise<Document> {
  const attributesFormat = hasFormatOfType(meta.options.formats, "attributes");

  if (!attributesFormat) {
    return document;
  }

  if (document.html === undefined) {
    throw new Error(
      "html is undefined -- this transformer is being called out of order",
    );
  }

  if (attributesFormat.selectors && attributesFormat.selectors.length > 0) {
    try {
      const attributes = await extractAttributes(
        document.html,
        attributesFormat.selectors,
      );

      if (attributes.length > 0) {
        document.attributes = attributes;

        meta.logger.debug("Extracted attributes", {
          count: attributes.length,
          attributes: attributes.map(d => ({
            selector: d.selector,
            attribute: d.attribute,
            valuesCount: d.values.length,
          })),
        });
      }
    } catch (error) {
      meta.logger.error("Failed to extract attributes", {
        error,
        url: meta.url,
      });
    }
  }

  return document;
}