Spaces:
Sleeping
Sleeping
File size: 7,169 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 | package com.firecrawl;
import com.firecrawl.client.FirecrawlClient;
import com.firecrawl.errors.FirecrawlException;
import com.firecrawl.models.Document;
import com.firecrawl.models.ScrapeOptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* Scrape Endpoint Tests
*
* Tests the scrape functionality of the Firecrawl Java SDK.
* These tests require FIRECRAWL_API_KEY environment variable to be set.
*
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.ScrapeTest"
*/
class ScrapeTest {
private static FirecrawlClient client;
@BeforeAll
static void setup() {
// Initialize client from environment variable
String apiKey = System.getenv("FIRECRAWL_API_KEY");
if (apiKey != null && !apiKey.isBlank()) {
client = FirecrawlClient.fromEnv();
}
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeBasic() {
// Test basic scraping with markdown format
System.out.println("Testing basic scrape with markdown format...");
Document doc = client.scrape("https://example.com",
ScrapeOptions.builder()
.formats(List.of("markdown"))
.build());
// Assertions
assertNotNull(doc, "Document should not be null");
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
assertFalse(doc.getMarkdown().isEmpty(), "Markdown content should not be empty");
System.out.println("β Basic scrape test passed");
System.out.println(" Markdown length: " + doc.getMarkdown().length() + " characters");
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeWithMultipleFormats() {
// Test scraping with multiple formats
System.out.println("Testing scrape with multiple formats (markdown + html)...");
Document doc = client.scrape("https://example.com",
ScrapeOptions.builder()
.formats(List.of("markdown", "html"))
.build());
// Assertions
assertNotNull(doc, "Document should not be null");
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
assertNotNull(doc.getHtml(), "HTML content should not be null");
assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
assertFalse(doc.getHtml().isEmpty(), "HTML should not be empty");
System.out.println("β Multiple formats test passed");
System.out.println(" Markdown length: " + doc.getMarkdown().length());
System.out.println(" HTML length: " + doc.getHtml().length());
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeWithMetadata() {
// Test that metadata is properly extracted
System.out.println("Testing scrape with metadata extraction...");
Document doc = client.scrape("https://example.com",
ScrapeOptions.builder()
.formats(List.of("markdown"))
.build());
// Assertions
assertNotNull(doc.getMetadata(), "Metadata should not be null");
assertNotNull(doc.getMetadata().get("sourceURL"), "Source URL should be in metadata");
assertTrue(doc.getMetadata().get("sourceURL").toString().contains("example.com"),
"Source URL should contain example.com");
System.out.println("β Metadata extraction test passed");
System.out.println(" Source URL: " + doc.getMetadata().get("sourceURL"));
if (doc.getMetadata().get("title") != null) {
System.out.println(" Title: " + doc.getMetadata().get("title"));
}
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeWithOnlyMainContent() {
// Test scraping with onlyMainContent option
System.out.println("Testing scrape with onlyMainContent option...");
Document doc = client.scrape("https://example.com",
ScrapeOptions.builder()
.formats(List.of("markdown"))
.onlyMainContent(true)
.build());
// Assertions
assertNotNull(doc, "Document should not be null");
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
System.out.println("β Only main content test passed");
System.out.println(" Content length: " + doc.getMarkdown().length());
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeWithTimeout() {
// Test scraping with custom timeout
System.out.println("Testing scrape with custom timeout...");
Document doc = client.scrape("https://example.com",
ScrapeOptions.builder()
.formats(List.of("markdown"))
.timeout(10000) // 10 seconds
.build());
// Assertions
assertNotNull(doc, "Document should not be null");
assertNotNull(doc.getMarkdown(), "Markdown should not be null");
System.out.println("β Timeout configuration test passed");
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeInvalidUrl() {
// Test that invalid URLs are handled properly
System.out.println("Testing scrape with invalid URL...");
assertThrows(FirecrawlException.class, () -> {
client.scrape("not-a-valid-url",
ScrapeOptions.builder()
.formats(List.of("markdown"))
.build());
}, "Should throw FirecrawlException for invalid URL");
System.out.println("β Invalid URL handling test passed");
}
@Test
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
void testScrapeWithWaitFor() {
// Test scraping with waitFor option (useful for dynamic content)
System.out.println("Testing scrape with waitFor option...");
Document doc = client.scrape("https://example.com",
ScrapeOptions.builder()
.formats(List.of("markdown"))
.waitFor(1000) // Wait 1 second for page to load
.build());
// Assertions
assertNotNull(doc, "Document should not be null");
assertNotNull(doc.getMarkdown(), "Markdown should not be null");
System.out.println("β WaitFor option test passed");
}
}
|