File size: 5,810 Bytes
097eaf0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
package com.firecrawl;

import com.firecrawl.client.FirecrawlClient;
import com.firecrawl.models.Document;
import com.firecrawl.models.ScrapeOptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;

import java.util.List;

import static org.junit.jupiter.api.Assertions.*;

/**
 * Live Site Test - Firecrawl.dev
 * 
 * Tests the Java SDK against the actual Firecrawl production website.
 * This demonstrates real-world usage of the API against live content.
 * 
 * Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.FirecrawlLiveSiteTest"
 */
class FirecrawlLiveSiteTest {

    private static FirecrawlClient client;

    @BeforeAll
    static void setup() {
        String apiKey = System.getenv("FIRECRAWL_API_KEY");
        if (apiKey != null && !apiKey.isBlank()) {
            client = FirecrawlClient.fromEnv();
        }
    }

    @Test
    @EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
    void testScrapeFirecrawlHomepage() {
        System.out.println("\n=== Testing against LIVE Firecrawl.dev website ===\n");
        System.out.println("Scraping: https://firecrawl.dev");
        
        Document doc = client.scrape("https://firecrawl.dev",
                ScrapeOptions.builder()
                        .formats(List.of("markdown", "html"))
                        .onlyMainContent(true)
                        .build());

        // Assertions
        assertNotNull(doc, "Document should not be null");
        assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
        assertNotNull(doc.getHtml(), "HTML content should not be null");
        assertNotNull(doc.getMetadata(), "Metadata should not be null");
        
        // Verify it's actually the Firecrawl site
        String markdown = doc.getMarkdown().toLowerCase();
        assertTrue(markdown.contains("firecrawl") || markdown.contains("scrape") || markdown.contains("crawl"),
                "Content should mention Firecrawl features");
        
        // Check metadata
        String sourceUrl = doc.getMetadata().get("sourceURL").toString();
        assertTrue(sourceUrl.contains("firecrawl.dev"), "Source URL should be firecrawl.dev");
        
        // Display results
        System.out.println("\n✓ Successfully scraped Firecrawl.dev!");
        System.out.println("\nMetadata:");
        System.out.println("  Source URL: " + sourceUrl);
        if (doc.getMetadata().get("title") != null) {
            System.out.println("  Title: " + doc.getMetadata().get("title"));
        }
        System.out.println("  Status Code: " + doc.getMetadata().get("statusCode"));
        
        System.out.println("\nContent Stats:");
        System.out.println("  Markdown length: " + doc.getMarkdown().length() + " characters");
        System.out.println("  HTML length: " + doc.getHtml().length() + " characters");
        
        System.out.println("\nFirst 500 characters of markdown:");
        System.out.println("  " + doc.getMarkdown().substring(0, Math.min(500, doc.getMarkdown().length())).replace("\n", "\n  "));
        
        System.out.println("\n=== Live site test completed successfully! ===\n");
    }

    @Test
    @EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
    void testScrapeFirecrawlPricing() {
        System.out.println("\n=== Testing Firecrawl Pricing Page ===\n");
        System.out.println("Scraping: https://firecrawl.dev/pricing");
        
        Document doc = client.scrape("https://firecrawl.dev/pricing",
                ScrapeOptions.builder()
                        .formats(List.of("markdown"))
                        .build());

        // Assertions
        assertNotNull(doc, "Document should not be null");
        assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
        
        String markdown = doc.getMarkdown().toLowerCase();
        assertTrue(markdown.contains("pricing") || markdown.contains("plan") || markdown.contains("price"),
                "Pricing page should contain pricing information");
        
        System.out.println("✓ Successfully scraped pricing page!");
        System.out.println("  Content length: " + doc.getMarkdown().length() + " characters");
        System.out.println("  Source: " + doc.getMetadata().get("sourceURL"));
    }

    @Test
    @EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
    void testScrapeFirecrawlDocs() {
        System.out.println("\n=== Testing Firecrawl Documentation ===\n");
        System.out.println("Scraping: https://docs.firecrawl.dev");
        
        Document doc = client.scrape("https://docs.firecrawl.dev",
                ScrapeOptions.builder()
                        .formats(List.of("markdown"))
                        .waitFor(2000)  // Wait for docs to load
                        .build());

        // Assertions
        assertNotNull(doc, "Document should not be null");
        assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
        assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
        
        String markdown = doc.getMarkdown().toLowerCase();
        assertTrue(markdown.contains("document") || markdown.contains("api") || markdown.contains("firecrawl"),
                "Docs should contain documentation content");
        
        System.out.println("✓ Successfully scraped documentation!");
        System.out.println("  Content length: " + doc.getMarkdown().length() + " characters");
        System.out.println("  Source: " + doc.getMetadata().get("sourceURL"));
        
        System.out.println("\n=== All Firecrawl.dev tests passed! ===\n");
    }
}