modules_search.js

// Browser Search & Scrape Module for HuggingBot
// Uses the Browser Search/Scrape API endpoint
// Endpoint: POST https://broadfield-dev-browser-api.hf.space/web_browse
// Contract (verified 2025):
//   action: "Search" | "Scrape"  (server also accepts "Scrape URL")
//   query: search term OR url to scrape
//   browser_name: "firefox"|"chromium"|"webkit"  (also tolerates "browser")
//   search_engine_name: string (also tolerates "search_engine")
// Response: { status, query, action, final_url, page_title, http_status, proxy_used, markdown_content }
//   -> markdown_content is the ONLY content field the endpoint returns.

(function(MS) {
  "use strict";

  var DEFAULT_API = "https://broadfield-dev-browser-api.hf.space/web_browse";
  var FALLBACK_URL = "https://duckduckgo.com/";

  function absUrl(u, base) {
    if (!u) return "";
    if (/^(https?:)?\/\//.test(u) === false && u.indexOf("://") === -1) {
      try { return new URL(u, base || FALLBACK_URL).href; } catch (e) { return u; }
    }
    return u;
  }

  // Decode DuckDuckGo-style redirect links: https://duckduckgo.com/l/?uddg=<enc>&rut=...
  function decodeRedirect(urlStr) {
    if (typeof urlStr !== "string") return "";
    try {
      if (urlStr.indexOf("uddg=") !== -1) {
        var m = urlStr.match(/[?&]uddg=([^&]+)/);
        if (m && m[1]) return decodeURIComponent(m[1]);
      }
      var mm = urlStr.match(/[?&]url=([^&]+)/);
      if (mm && mm[1]) return decodeURIComponent(mm[1]);
    } catch (e) {}
    return urlStr;
  }

  // Pull [title](url) pairs (and bare URLs) out of returned markdown.
  function linksFromMarkdown(md, base) {
    var out = [], seen = {}, m;
    var re = /\[([^\]]+)\]\((https?:\/\/[^\s)]+)\)/g;
    while ((m = re.exec(md)) !== null) {
      var t = (m[1] || "").trim();
      var u = decodeRedirect(m[2]);
      if (t && u && !seen[u]) { seen[u] = true; out.push({ title: t, url: u }); }
    }
    if (out.length === 0) {
      var re2 = /(https?:\/\/[^\s"'<>)\]]+)/g;
      var idx = 0;
      while ((m = re2.exec(md)) !== null && out.length < 10) {
        var u2 = decodeRedirect(m[1]);
        if (u2 && !seen[u2]) {
          seen[u2] = true;
          var before = md.substring(Math.max(0, m.index - 60), m.index).replace(/\n/g, " ").trim();
          if (before.length > 40) before = "..." + before.substring(before.length - 40);
          out.push({ title: before || ("Result " + (idx + 1)), url: u2 });
          idx++;
        }
      }
    }
    return out;
  }

  // Build our structured result list from the endpoint's markdown SERP.
  function extractSearchResults(mdContent, baseUrl, mx) {
    var results = [], lines = String(mdContent).split("\n"), i;
    for (i = 0; i < lines.length && results.length < mx; i++) {
      var line = lines[i].trim();
      if (!line) continue;
      var linkM = line.match(/\[([^\]]+)\]\((https?:\/\/[^\s)]+)\)/);
      if (linkM) {
        results.push({ title: linkM[1], snippet: "", url: decodeRedirect(linkM[2]) });
        continue;
      }
      if (line.indexOf("http") !== -1) {
        var urlM = line.match(/(https?:\/\/[^\s"'<>)\]]+)/);
        if (urlM) {
          var title = line.replace(urlM[1], "").replace(/^[#>\-*|\[\]]+\s*/, "").trim();
          if (!title && i > 0) title = lines[i - 1].trim();
          if (title.length > 60) title = title.substring(0, 60) + "\u2026";
          results.push({ title: title || ("Result " + (results.length + 1)), snippet: "", url: decodeRedirect(urlM[1]) });
        }
      }
    }
    if (results.length === 0) {
      var linkList = linksFromMarkdown(mdContent, baseUrl);
      results = linkList.slice(0, mx);
    }
    return results;
  }

  function apiPayload(action, query, cfg) {
    cfg = cfg || {};
    // Send BOTH key styles so the endpoint works regardless of which variant it reads.
    var p = { action: action, query: query, browser_name: cfg.browser_name || "chromium", browser: cfg.browser_name || "chromium", search_engine_name: cfg.search_engine || "DuckDuckGo", search_engine: cfg.search_engine || "DuckDuckGo" };
    return JSON.stringify(p);
  }

  async function postApi(action, query, timeoutMs, cfg) {
    var resp = await fetch(DEFAULT_API, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: apiPayload(action, query, cfg),
      signal: AbortSignal.timeout(timeoutMs || 30000)
    });
    if (!resp.ok) {
      var errText = "";
      try { errText = await resp.text(); } catch (e) {}
      var e = new Error("HTTP " + resp.status + (errText ? ": " + errText.substring(0, 200) : ""));
      e.status = resp.status;
      throw e;
    }
    return await resp.json();
  }

  MS.register("search-web", {
    name: "Web Search 2.0",
    description: "Browser-based web search and page scraping using the broadfield-dev API endpoint.",
    tools: [
      {
        type: "function",
        function: {
          name: "search_web",
          description: "Search the internet for current information. Uses a browser-based search API to fetch live results. Returns titles, snippets, and URLs.",
          parameters: {
            type: "object",
            properties: {
              query: { type: "string", description: "Search query - be specific for best results" },
              max_results: { type: "number", description: "Max results (1-10)", default: 5 },
              source: { type: "string", enum: ["auto", "web", "wikipedia", "news", "reddit"], description: "Source to search.", default: "auto" }
            },
            required: ["query"]
          }
        }
      },
      {
        type: "function",
        function: {
          name: "scrape_web",
          description: "Scrape/extract the full content of a webpage by URL. Uses a browser-based API to fetch and return the page's rendered content.",
          parameters: {
            type: "object",
            properties: {
              url: { type: "string", description: "The full URL of the webpage to scrape (e.g. https://example.com/page)" },
              max_length: { type: "number", description: "Maximum length of returned content in characters (100-50000)", default: 5000 }
            },
            required: ["url"]
          }
        }
      }
    ],
    handleToolCall: async function(tn, args) {
      if (tn === "search_web") {
        try {
          var q = args.query;
          var mx = Math.min(Math.max(parseInt(args.max_results) || 5, 1), 10);
          var allResults = [];

          // 1) Primary: broadfield-dev browser API
          try {
            var data = await postApi("Search", q, 30000, { browser_name: "chromium", search_engine: "DuckDuckGo" });
            if (data && data.status === "success") {
              if (typeof data.markdown_content === "string" && data.markdown_content.length > 0) {
                var baseUrl = data.final_url || FALLBACK_URL;
                var parsed = extractSearchResults(data.markdown_content, baseUrl, mx);
                if (parsed.length > 0) {
                  allResults = parsed;
                } else {
                  allResults.push({ title: data.page_title || "Search Result", snippet: data.markdown_content.substring(0, 300), url: data.final_url || "", source: "API" });
                }
              } else if (data.results && Array.isArray(data.results)) {
                for (var i = 0; i < data.results.length && allResults.length < mx; i++) {
                  var r = data.results[i];
                  allResults.push({ title: r.title || r.name || "", snippet: r.snippet || r.description || r.summary || "", url: absUrl(decodeRedirect(r.url || r.link || r.href || ""), data.final_url), source: "Web" });
                }
              } else if (data.items && Array.isArray(data.items)) {
                for (var i = 0; i < data.items.length && allResults.length < mx; i++) {
                  var it = data.items[i];
                  allResults.push({ title: it.title || it.name || "", snippet: it.snippet || it.description || it.summary || "", url: absUrl(decodeRedirect(it.link || it.url || it.href || ""), data.final_url), source: "Web" });
                }
              } else {
                allResults.push({ title: "Search Result", snippet: JSON.stringify(data).substring(0, 400), url: data.final_url || "", source: "Web" });
              }
            } else {
              allResults.push({ title: "API Error", snippet: (data && data.error_message) || "Unknown API error", url: "", source: "API" });
            }
          } catch (e) {
            allResults.push({ title: "Connection Error", snippet: "Could not reach the browser search API: " + e.message, url: "", source: "Web" });
          }

          // 2) Fallback: DuckDuckGo (only if primary failed or returned nothing)
          if (allResults.length === 0 || allResults[0].title === "API Error" || allResults[0].title === "Connection Error" || allResults[0].title === "Search Result") {
            try {
              var ddgUrl = "https://api.duckduckgo.com/?q=" + encodeURIComponent(q) + "&format=json&no_html=1&skip_disambig=1";
              var ddgResp = await fetch(ddgUrl, { method: "GET", signal: AbortSignal.timeout(10000) });
              if (ddgResp.ok) {
                var ddgData = await ddgResp.json();
                var ddgResults = [];
                if (ddgData.AbstractText) ddgResults.push({ title: ddgData.Heading || "Summary", snippet: ddgData.AbstractText, url: ddgData.AbstractURL || "", source: "DuckDuckGo" });
                var topics = ddgData.RelatedTopics || [];
                for (var i = 0; i < topics.length && ddgResults.length < mx; i++) {
                  var topic = topics[i];
                  if (topic.Topics && Array.isArray(topic.Topics)) {
                    for (var j = 0; j < topic.Topics.length && ddgResults.length < mx; j++) {
                      var sub = topic.Topics[j];
                      if (sub.Text || sub.FirstURL) ddgResults.push({ title: (sub.Text || "").split(" - ")[0], snippet: sub.Text || "", url: sub.FirstURL || "", source: "DuckDuckGo" });
                    }
                  } else if (topic.Text || topic.FirstURL) {
                    ddgResults.push({ title: (topic.Text || "").split(" - ")[0], snippet: topic.Text || "", url: topic.FirstURL || "", source: "DuckDuckGo" });
                  }
                }
                if (ddgResults.length > 0) {
                  allResults = ddgResults;
                } else {
                  var htmlResp = await fetch("https://html.duckduckgo.com/html/?q=" + encodeURIComponent(q), { method: "GET", signal: AbortSignal.timeout(10000) });
                  if (htmlResp.ok) {
                    var htmlText = await htmlResp.text();
                    var linkRegex = /<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi;
                    var snippetRegex = /<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
                    var linkMatches = [], snippetMatches = [], lm, sm;
                    while ((lm = linkRegex.exec(htmlText)) !== null) linkMatches.push({ url: lm[1], title: lm[2].replace(/<[^>]*>/g, "").trim() });
                    while ((sm = snippetRegex.exec(htmlText)) !== null) snippetMatches.push(sm[1].replace(/<[^>]*>/g, "").trim());
                    var htmlResults = [];
                    for (var i = 0; i < linkMatches.length && htmlResults.length < mx; i++) {
                      htmlResults.push({ title: linkMatches[i].title, snippet: snippetMatches[i] || "", url: decodeRedirect(linkMatches[i].url), source: "DuckDuckGo" });
                    }
                    if (htmlResults.length > 0) allResults = htmlResults;
                  }
                }
              }
            } catch (e) {
              if (allResults.length === 0 || allResults[0].title === "API Error" || allResults[0].title === "Connection Error") {
                allResults = [{ title: "Search Unavailable", snippet: "Both search methods are currently unavailable. Please try again later.", url: "", source: "Web" }];
              }
            }
          }

          // Dedupe + format
          var seen = {}, deduped = [];
          for (var i = 0; i < allResults.length && deduped.length < mx; i++) {
            var item = allResults[i];
            if (!item.title) continue;
            var key = item.url || item.snippet || item.title;
            if (key && !seen[key.substring(0, 100)]) { seen[key.substring(0, 100)] = true; deduped.push(item); }
          }
          var out = ["## \uD83D\uDD0D Search Results: \"" + q + "\"", ""];
          if (deduped.length === 0) {
            out.push("*No results found. Try a different query.*");
          } else {
            for (var i = 0; i < deduped.length; i++) {
              out.push("### " + (i + 1) + ". " + deduped[i].title);
              if (deduped[i].snippet) out.push("> " + deduped[i].snippet);
              if (deduped[i].url) out.push("\uD83D\uDD17 " + deduped[i].url);
              out.push("");
            }
          }
          return out.join("\n");
        } catch (e) {
          return "## \uD83D\uDD0D Search: \"" + args.query + "\"\n\n\u274C Error: " + e.message;
        }
      }

      if (tn === "scrape_web") {
        try {
          var urlToScrape = args.url;
          var maxLen = Math.min(Math.max(parseInt(args.max_length) || 5000, 100), 50000);
          var out = ["## \uD83D\uDCC4 Scrape: " + urlToScrape, ""];

          // Primary action: "Scrape" (proven to work); fallback to "Scrape URL" if needed.
          try {
            var data = null;
            try {
              data = await postApi("Scrape", urlToScrape, 60000, { browser_name: "chromium" });
            } catch (e2) {
              data = await postApi("Scrape URL", urlToScrape, 60000, { browser_name: "chromium" });
            }

            if (data) {
              if (data.status === "error") {
                out.push("\u274C **API Error:** " + (data.error_message || "Unknown error"));
              } else {
                var content = "";
                if (typeof data.markdown_content === "string") content = data.markdown_content;
                else if (typeof data.text === "string") content = data.text;
                else if (typeof data.content === "string") content = data.content;
                else if (typeof data.html === "string") content = data.html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/g, " ").replace(/\s+/g, " ").trim();
                else if (typeof data.body === "string") content = data.body.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
                else if (data.title || data.description) content = "Title: " + (data.title || "") + "\nDescription: " + (data.description || "");
                else { content = JSON.stringify(data, null, 2); if (content.length > maxLen * 2) content = content.substring(0, maxLen * 2) + "\n... [truncated]"; }

                if (content.length > maxLen) content = content.substring(0, maxLen) + "\n\n... [Content truncated. Use a higher max_length to get more.]";

                if (content && content.trim().length > 0) {
                  if (data.page_title) out.push("**Title:** " + data.page_title);
                  if (data.final_url) out.push("**URL:** " + data.final_url);
                  out.push("");
                  out.push(content);
                } else {
                  out.push("*No readable content could be extracted from this page.*");
                  out.push("");
                  out.push("Raw response keys: " + Object.keys(data).join(", "));
                }
              }
            } else {
              out.push("\u274C **API Error:** Empty response from scrape API");
            }
          } catch (e) {
            out.push("\u274C **Connection Error:** Could not reach the scrape API: " + e.message);
          }

          return out.join("\n");
        } catch (e) {
          return "## \uD83D\uDCC4 Scrape Error\n\n\u274C Error: " + e.message;
        }
      }

      return null;
    }
  });
})(ModuleSystem);