File size: 6,821 Bytes
097eaf0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
# frozen_string_literal: true

require "net/http"
require "json"
require "securerandom"
require "uri"

module Firecrawl
  # Internal HTTP client for making authenticated requests to the Firecrawl API.
  # Handles retry logic with exponential backoff.
  #
  # @api private
  class HttpClient
    def initialize(api_key:, base_url:, timeout:, max_retries:, backoff_factor:)
      @api_key = api_key
      @base_url = base_url.chomp("/")
      @timeout = timeout
      @max_retries = max_retries
      @backoff_factor = backoff_factor
    end

    # Sends a POST request with JSON body.
    def post(path, body, extra_headers: {})
      uri = URI("#{@base_url}#{path}")
      request = Net::HTTP::Post.new(uri)
      request["Authorization"] = "Bearer #{@api_key}"
      request["Content-Type"] = "application/json"
      extra_headers.each { |k, v| request[k] = v }
      request.body = JSON.generate(body)
      execute_with_retry(uri, request)
    end

    # Sends a GET request.
    def get(path)
      uri = URI("#{@base_url}#{path}")
      request = Net::HTTP::Get.new(uri)
      request["Authorization"] = "Bearer #{@api_key}"
      execute_with_retry(uri, request)
    end

    # Sends a GET request with a full URL (for following next-page cursors).
    # Only sends the Authorization header if the URL matches the configured API origin.
    def get_absolute(absolute_url)
      uri = URI(absolute_url)
      base_uri = URI(@base_url)
      unless uri.host == base_uri.host && uri.port == base_uri.port && uri.scheme == base_uri.scheme
        raise FirecrawlError, "Absolute URL origin (#{uri.scheme}://#{uri.host}:#{uri.port}) does not match API base URL origin (#{base_uri.scheme}://#{base_uri.host}:#{base_uri.port}). Refusing to send credentials."
      end
      request = Net::HTTP::Get.new(uri)
      request["Authorization"] = "Bearer #{@api_key}"
      execute_with_retry(uri, request)
    end

    # Sends a DELETE request.
    def delete(path)
      uri = URI("#{@base_url}#{path}")
      request = Net::HTTP::Delete.new(uri)
      request["Authorization"] = "Bearer #{@api_key}"
      execute_with_retry(uri, request)
    end

    # Sends a PATCH request with JSON body.
    def patch(path, body)
      uri = URI("#{@base_url}#{path}")
      request = Net::HTTP::Patch.new(uri)
      request["Authorization"] = "Bearer #{@api_key}"
      request["Content-Type"] = "application/json"
      request.body = JSON.generate(body)
      execute_with_retry(uri, request)
    end

    # Sends a POST request with a multipart/form-data body.
    #
    # @param path [String] API path
    # @param fields [Hash{String=>String}] additional form fields to include
    # @param file_field [String] form field name for the file part (e.g. "file")
    # @param filename [String] filename to send with the file part
    # @param content [String] raw bytes for the file part
    # @param content_type [String, nil] optional MIME type for the file part
    def post_multipart(path, fields:, file_field:, filename:, content:, content_type: nil)
      uri = URI("#{@base_url}#{path}")
      boundary = "----FirecrawlBoundary#{SecureRandom.hex(16)}"
      body = build_multipart_body(boundary, fields, file_field, filename, content, content_type)

      builder = lambda do
        request = Net::HTTP::Post.new(uri)
        request["Authorization"] = "Bearer #{@api_key}"
        request["Content-Type"] = "multipart/form-data; boundary=#{boundary}"
        request.body = body
        request
      end

      execute_with_retry(uri, builder.call, request_builder: builder)
    end

    private

    def build_multipart_body(boundary, fields, file_field, filename, content, content_type)
      parts = +""
      fields.each do |name, value|
        parts << "--#{boundary}\r\n"
        parts << %(Content-Disposition: form-data; name="#{name}"\r\n\r\n)
        parts << value.to_s
        parts << "\r\n"
      end

      parts << "--#{boundary}\r\n"
      safe_file_field = file_field.to_s.gsub(/[\r\n"]/, "_")
      safe_filename = filename.to_s.gsub(/[\r\n"]/, "_")
      parts << %(Content-Disposition: form-data; name="#{safe_file_field}"; filename="#{safe_filename}"\r\n)
      parts << "Content-Type: #{content_type || "application/octet-stream"}\r\n\r\n"
      parts.force_encoding(Encoding::ASCII_8BIT)
      parts << content.to_s.dup.force_encoding(Encoding::ASCII_8BIT)
      parts << "\r\n--#{boundary}--\r\n"
      parts
    end

    def execute_with_retry(uri, request, request_builder: nil)
      attempt = 0
      loop do
        response = perform_request(uri, request)
        code = response.code.to_i
        body_str = response.body || ""

        if code >= 200 && code < 300
          return body_str.empty? ? {} : JSON.parse(body_str)
        end

        error_message = extract_error_message(body_str, code)
        error_code = extract_error_code(body_str)

        # Non-retryable client errors
        if code == 401
          raise AuthenticationError.new(error_message, error_code: error_code)
        end
        if code == 429
          raise RateLimitError.new(error_message, error_code: error_code)
        end
        if code >= 400 && code < 500 && code != 408 && code != 409
          raise FirecrawlError.new(error_message, status_code: code, error_code: error_code)
        end

        # Retryable errors: 408, 409, 502, 5xx
        if attempt < @max_retries
          attempt += 1
          sleep_with_backoff(attempt)
          request = request_builder.call if request_builder
          next
        end

        raise FirecrawlError.new(error_message, status_code: code, error_code: error_code)
      rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT,
             Net::OpenTimeout, Net::ReadTimeout, IOError => e
        if attempt < @max_retries
          attempt += 1
          sleep_with_backoff(attempt)
          request = request_builder.call if request_builder
          retry
        end
        raise FirecrawlError.new("Request failed: #{e.message}")
      end
    end

    def perform_request(uri, request)
      http = Net::HTTP.new(uri.host, uri.port)
      http.use_ssl = (uri.scheme == "https")
      http.open_timeout = @timeout
      http.read_timeout = @timeout
      http.write_timeout = @timeout
      http.request(request)
    end

    def extract_error_message(body, status_code)
      parsed = JSON.parse(body)
      parsed["error"] || parsed["message"] || "HTTP #{status_code} error"
    rescue JSON::ParserError
      "HTTP #{status_code} error"
    end

    def extract_error_code(body)
      parsed = JSON.parse(body)
      code = parsed["code"]
      code&.to_s
    rescue JSON::ParserError
      nil
    end

    def sleep_with_backoff(attempt)
      delay = @backoff_factor * (2**(attempt - 1))
      sleep(delay)
    end
  end
end