Spaces:
Sleeping
Sleeping
File size: 5,810 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | package com.firecrawl;
import com.firecrawl.client.FirecrawlClient;
import com.firecrawl.models.Document;
import com.firecrawl.models.ScrapeOptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* Live Site Test - Firecrawl.dev
*
* Tests the Java SDK against the actual Firecrawl production website.
* This demonstrates real-world usage of the API against live content.
*
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.FirecrawlLiveSiteTest"
*/
class FirecrawlLiveSiteTest {
private static FirecrawlClient client;
@BeforeAll
static void setup() {
String apiKey = System.getenv("FIRECRAWL_API_KEY");
if (apiKey != null && !apiKey.isBlank()) {
client = FirecrawlClient.fromEnv();
}
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeFirecrawlHomepage() {
System.out.println("\n=== Testing against LIVE Firecrawl.dev website ===\n");
System.out.println("Scraping: https://firecrawl.dev");
Document doc = client.scrape("https://firecrawl.dev",
ScrapeOptions.builder()
.formats(List.of("markdown", "html"))
.onlyMainContent(true)
.build());
// Assertions
assertNotNull(doc, "Document should not be null");
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
assertNotNull(doc.getHtml(), "HTML content should not be null");
assertNotNull(doc.getMetadata(), "Metadata should not be null");
// Verify it's actually the Firecrawl site
String markdown = doc.getMarkdown().toLowerCase();
assertTrue(markdown.contains("firecrawl") || markdown.contains("scrape") || markdown.contains("crawl"),
"Content should mention Firecrawl features");
// Check metadata
String sourceUrl = doc.getMetadata().get("sourceURL").toString();
assertTrue(sourceUrl.contains("firecrawl.dev"), "Source URL should be firecrawl.dev");
// Display results
System.out.println("\n✓ Successfully scraped Firecrawl.dev!");
System.out.println("\nMetadata:");
System.out.println(" Source URL: " + sourceUrl);
if (doc.getMetadata().get("title") != null) {
System.out.println(" Title: " + doc.getMetadata().get("title"));
}
System.out.println(" Status Code: " + doc.getMetadata().get("statusCode"));
System.out.println("\nContent Stats:");
System.out.println(" Markdown length: " + doc.getMarkdown().length() + " characters");
System.out.println(" HTML length: " + doc.getHtml().length() + " characters");
System.out.println("\nFirst 500 characters of markdown:");
System.out.println(" " + doc.getMarkdown().substring(0, Math.min(500, doc.getMarkdown().length())).replace("\n", "\n "));
System.out.println("\n=== Live site test completed successfully! ===\n");
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeFirecrawlPricing() {
System.out.println("\n=== Testing Firecrawl Pricing Page ===\n");
System.out.println("Scraping: https://firecrawl.dev/pricing");
Document doc = client.scrape("https://firecrawl.dev/pricing",
ScrapeOptions.builder()
.formats(List.of("markdown"))
.build());
// Assertions
assertNotNull(doc, "Document should not be null");
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
String markdown = doc.getMarkdown().toLowerCase();
assertTrue(markdown.contains("pricing") || markdown.contains("plan") || markdown.contains("price"),
"Pricing page should contain pricing information");
System.out.println("✓ Successfully scraped pricing page!");
System.out.println(" Content length: " + doc.getMarkdown().length() + " characters");
System.out.println(" Source: " + doc.getMetadata().get("sourceURL"));
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeFirecrawlDocs() {
System.out.println("\n=== Testing Firecrawl Documentation ===\n");
System.out.println("Scraping: https://docs.firecrawl.dev");
Document doc = client.scrape("https://docs.firecrawl.dev",
ScrapeOptions.builder()
.formats(List.of("markdown"))
.waitFor(2000) // Wait for docs to load
.build());
// Assertions
assertNotNull(doc, "Document should not be null");
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
String markdown = doc.getMarkdown().toLowerCase();
assertTrue(markdown.contains("document") || markdown.contains("api") || markdown.contains("firecrawl"),
"Docs should contain documentation content");
System.out.println("✓ Successfully scraped documentation!");
System.out.println(" Content length: " + doc.getMarkdown().length() + " characters");
System.out.println(" Source: " + doc.getMetadata().get("sourceURL"));
System.out.println("\n=== All Firecrawl.dev tests passed! ===\n");
}
}
|