import os, re, json, time, sqlite3, subprocess, importlib.util, base64, socket, ipaddress, threading, shutil from urllib.parse import urlparse import requests from bs4 import BeautifulSoup WORKSPACE = os.environ.get("AGENT_WORKSPACE", "/tmp/agent_ws") LOG_DIR = os.path.join(WORKSPACE, "logs") BACKUP_DIR = os.path.join(WORKSPACE, "backups") TOOLS_CUSTOM_DIR = os.path.join(WORKSPACE, "tools_custom") for d in (WORKSPACE, LOG_DIR, BACKUP_DIR, TOOLS_CUSTOM_DIR): os.makedirs(d, exist_ok=True) DB_PATH = os.path.join(WORKSPACE, "knowledge.db") AUDIT_PATH = os.path.join(WORKSPACE, "audit.json") INDEX_PATH = os.path.join(LOG_DIR, "index.json") REPO_DIR = os.environ.get("REPO_DIR", "/app") SPACE_ID = os.environ.get("SPACE_ID", "satyadev13/quess") GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "") HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_HUB_TOKEN", "") UA = {"User-Agent": "Mozilla/5.0 (private-research-agent)"} _LOCK = threading.Lock() # ================= REDAKSI SECRET (pra-upload log) ================= def _collect_secrets(): vals = set() for k in ("GITHUB_TOKEN", "HF_TOKEN", "HUGGINGFACE_HUB_TOKEN", "ADMIN_PASS", "ADMIN_USER"): v = os.environ.get(k) if v and len(v) > 5: vals.add(v) try: db = json.load(open(os.path.join(REPO_DIR, "database.json"))) for k in db.get("api_keys", {}): vals.add(k) except Exception: pass return sorted(vals, key=len, reverse=True) SECRETS = _collect_secrets() def redact(t): for s in SECRETS: t = t.replace(s, "[REDACTED]") return t # ================= ACT LOG: riwayat UTUH per request ================= class ActLog: def __init__(self, act_id): self.id = act_id self.path = os.path.join(LOG_DIR, f"{act_id}.jsonl") def write(self, event): with _LOCK: with open(self.path, "a") as f: f.write(json.dumps(event, default=str) + "\n") def read(self, offset=0, limit=50): if not os.path.exists(self.path): return [] with open(self.path) as f: lines = f.readlines()[offset:offset + limit] return [json.loads(l) for l in lines] def history_list(): idx = json.load(open(INDEX_PATH)) if os.path.exists(INDEX_PATH) else {} return {"archived": idx, "local": [f[:-6] for f in os.listdir(LOG_DIR) if f.endswith(".jsonl")]} def history_read(act_id, offset=0, limit=50): return ActLog(act_id).read(offset, limit) def history_summarize(act_id, summary): idx = json.load(open(INDEX_PATH)) if os.path.exists(INDEX_PATH) else {} idx.setdefault(act_id, {})["summary"] = summary json.dump(idx, open(INDEX_PATH, "w"), indent=1) return {"status": "summary saved", "act_id": act_id} def history_fetch(url, max_chars=20000): r = requests.get(url, headers=UA, timeout=60) r.raise_for_status() return {"content": r.text[:max_chars]} def _catbox_upload(path): with open(path, "rb") as f: r = requests.post("https://catbox.moe/user/api.php", data={"reqtype": "fileupload"}, files={"fileToUpload": (os.path.basename(path), f)}, timeout=120) u = r.text.strip() return u if u.startswith("http") else None def history_finalize(act_id): log = ActLog(act_id) if not os.path.exists(log.path): return {"status": "empty"} tmp = log.path + ".redacted" with open(log.path) as f: open(tmp, "w").write(redact(f.read())) url = _catbox_upload(tmp) idx = json.load(open(INDEX_PATH)) if os.path.exists(INDEX_PATH) else {} if url: os.remove(log.path) os.remove(tmp) idx[act_id] = {"url": url, "ts": time.time()} json.dump(idx, open(INDEX_PATH, "w"), indent=1) return {"status": "archived", "url": url} os.remove(tmp) return {"status": "upload gagal, log lokal dipertahankan", "path": log.path} # ================= MEMORY (SQLite FTS5) ================= def _db(): con = sqlite3.connect(DB_PATH) con.execute("CREATE VIRTUAL TABLE IF NOT EXISTS knowledge USING fts5(title, content, source, ts)") return con def mem_save(title, content, source=""): with _LOCK: con = _db() con.execute("INSERT INTO knowledge(title,content,source,ts) VALUES(?,?,?,?)", (title, content, source, time.strftime("%Y-%m-%d %H:%M:%S"))) con.commit() con.close() return {"status": "saved"} def mem_recall(query, limit=5): con = _db() try: rows = con.execute( "SELECT title, snippet(knowledge,1,'[',']','...',32), source FROM knowledge " "WHERE knowledge MATCH ? ORDER BY rank LIMIT ?", (query, limit)).fetchall() except Exception: like = f"%{query}%" rows = con.execute( "SELECT title, substr(content,1,200), source FROM knowledge " "WHERE title LIKE ? OR content LIKE ? LIMIT ?", (like, like, limit)).fetchall() con.close() return [{"title": r[0], "snippet": r[1], "source": r[2]} for r in rows] # ================= WEB ================= def _safe_url(url): u = urlparse(url) if u.scheme not in ("http", "https") or not u.hostname: return False try: ip = ipaddress.ip_address(socket.gethostbyname(u.hostname)) return not (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved) except Exception: return False def web_search(query, max_results=5): try: r = requests.post("https://html.duckduckgo.com/html/", data={"q": query}, headers=UA, timeout=20) soup = BeautifulSoup(r.text, "html.parser") out = [{"title": a.get_text(strip=True), "url": a.get("href", "")} for a in soup.select("a.result__a")[:max_results]] return out or [{"error": "no results"}] except Exception as e: return [{"error": str(e)}] def web_scrape(url, max_chars=8000): if not _safe_url(url): return {"error": "URL diblok policy SSRF"} r = requests.get(url, headers=UA, timeout=30) r.raise_for_status() if "html" not in r.headers.get("content-type", ""): return {"content": r.text[:max_chars]} soup = BeautifulSoup(r.text, "html.parser") for t in soup(["script", "style", "nav", "footer", "header"]): t.decompose() return {"content": soup.get_text("\n", strip=True)[:max_chars]} # ================= CODE EXEC (BEBAS: fs + env + jaringan penuh) ================= def code_exec(code, timeout=120, install=""): if install: subprocess.run(["pip", "install", "--quiet", install], timeout=300, capture_output=True, cwd=WORKSPACE) try: p = subprocess.run(["python", "-c", code], cwd=WORKSPACE, capture_output=True, text=True, timeout=timeout) return {"stdout": p.stdout[-6000:], "stderr": p.stderr[-3000:], "rc": p.returncode} except subprocess.TimeoutExpired: return {"error": f"timeout {timeout}s", "rc": -1} # ================= GITHUB ================= def _gh(method, path, **kw): r = requests.request(method, f"https://api.github.com{path}", timeout=30, headers={"Authorization": f"token {GITHUB_TOKEN}", "Accept": "application/vnd.github+json"}, **kw) try: body = r.json() except Exception: body = r.text[:500] return {"status": r.status_code, "body": body} def gh_list_repos(): return _gh("GET", "/user/repos?per_page=100") def gh_search(query): return _gh("GET", f"/search/repositories?q={requests.utils.quote(query)}") def gh_create_repo(name, private=True): return _gh("POST", "/user/repos", json={"name": name, "private": private}) def gh_push_file(repo, path, content, message="agent commit"): cur = _gh("GET", f"/repos/{repo}/contents/{path}") payload = {"message": message, "content": base64.b64encode(content.encode()).decode()} if cur["status"] == 200 and isinstance(cur["body"], dict) and cur["body"].get("sha"): payload["sha"] = cur["body"]["sha"] return _gh("PUT", f"/repos/{repo}/contents/{path}", json=payload) def gh_create_action(repo, name, yaml_content): return gh_push_file(repo, f".github/workflows/{name}.yml", yaml_content, f"agent: workflow {name}") def gh_delete_repo(repo): return _gh("DELETE", f"/repos/{repo}") # ================= MODIFIKASI DIRI (bebas + snapshot otomatis) ================= CUSTOM_TOOLS = {} def load_custom_tools(): CUSTOM_TOOLS.clear() for fn in os.listdir(TOOLS_CUSTOM_DIR): if not fn.endswith(".py"): continue name = fn[:-3] spec = importlib.util.spec_from_file_location(name, os.path.join(TOOLS_CUSTOM_DIR, fn)) mod = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(mod) for attr in dir(mod): o = getattr(mod, attr) if callable(o) and getattr(o, "__module__", "") == mod.__name__ and not attr.startswith("_"): CUSTOM_TOOLS["custom_" + attr] = o except Exception as e: CUSTOM_TOOLS[f"custom_{name}_LOAD_ERROR"] = lambda *a, e=str(e), **k: {"error": e} load_custom_tools() def _repo_path(rel): return os.path.normpath(os.path.join(REPO_DIR, rel)) def self_list(rel=""): p = _repo_path(rel) if rel else REPO_DIR if not os.path.isdir(p): return {"error": "not a dir"} out = [] for root, dirs, files in os.walk(p): dirs[:] = [d for d in dirs if not d.startswith(".")] for f in files: out.append(os.path.relpath(os.path.join(root, f), REPO_DIR)) return {"files": out[:300]} def self_read(rel, max_chars=20000): p = _repo_path(rel) if not os.path.isfile(p): return {"error": "not found"} return {"content": open(p, errors="replace").read()[:max_chars]} def self_edit(rel, content): p = _repo_path(rel) os.makedirs(os.path.dirname(p) or ".", exist_ok=True) if os.path.exists(p): shutil.copy2(p, os.path.join(BACKUP_DIR, os.path.basename(p) + f".{int(time.time())}.bak")) with open(p, "w") as f: f.write(content) return {"status": "written", "path": p, "note": "perlu self_restart agar berlaku untuk app.py/agent_core.py"} def self_rollback(rel): p = _repo_path(rel) b = os.path.basename(p) c = sorted([f for f in os.listdir(BACKUP_DIR) if f.startswith(b + ".")]) if not c: return {"error": "no backup"} shutil.copy2(os.path.join(BACKUP_DIR, c[-1]), p) return {"status": "rolled back", "from": c[-1]} def self_restart(): r = requests.post(f"https://huggingface.co/api/spaces/{SPACE_ID}/restart", headers={"Authorization": f"Bearer {HF_TOKEN}"}, timeout=30) return {"status": r.status_code} # ================= OVERSIGHT ADVISORY (think-twice, tanpa gerbang manusia) ================= def oversight_note(tool, args): if tool == "gh_delete_repo": return "DESTRUKTIF IRREVERSIBEL: repo hilang permanen dari GitHub." if tool == "self_edit": return "MODIFIKASI DIRI: snapshot backup dibuat otomatis; self_rollback tersedia." if tool == "self_restart": return "Restart memutus SEMUA job yang sedang berjalan." if tool == "code_exec" and re.search(r"rm\s+-rf\s+/|shutil\.rmtree\(['\"]/|mkfs", str(args.get("code", ""))): return "POLA MERUSAK SISTEM: kontainer bisa mati total dan Space rebuild." return None # ================= REGISTRY ================= BASE_TOOLS = { "web_search": web_search, "web_scrape": web_scrape, "code_exec": code_exec, "mem_save": mem_save, "mem_recall": mem_recall, "history_list": history_list, "history_read": history_read, "history_summarize": history_summarize, "history_fetch": history_fetch, "gh_list_repos": gh_list_repos, "gh_search": gh_search, "gh_create_repo": gh_create_repo, "gh_push_file": gh_push_file, "gh_create_action": gh_create_action, "gh_delete_repo": gh_delete_repo, "self_list": self_list, "self_read": self_read, "self_edit": self_edit, "self_rollback": self_rollback, "self_restart": self_restart, } def ALL_TOOLS(): return {**BASE_TOOLS, **CUSTOM_TOOLS} TOOL_DOCS = ("web_search(query)|web_scrape(url)|code_exec(code,timeout,install) bebas baca fs/env| " "mem_save(title,content,source)|mem_recall(query)| " "history_list()|history_read(act_id,offset,limit)|history_summarize(act_id,summary)|history_fetch(url)| " "gh_list_repos()|gh_search(query)|gh_create_repo(name)|gh_push_file(repo,path,content,message)|" "gh_create_action(repo,name,yaml)|gh_delete_repo(repo)| " "self_list(rel)|self_read(rel)|self_edit(rel,content)|self_rollback(rel)|self_restart()| custom_*") SYSTEM_PROMPT = f"""Kamu adalah Quess-Agent v2, agen riset otonom tanpa pengawas manusia. Kejar tujuan sampai tuntas: cari, scrape, eksekusi kode, belajar, simpan, ulangi. Jangan menyerah pada kegagalan tool pertama; coba jalur lain. Riwayat langkahmu saat ini tersimpan UTUH di act log; baca potongan mana pun via history_read. Ringkas via history_summarize tanpa menghapus asli. Jika menerima RISK_NOTE, wajib berpikir ulang secara matang: kirim ulang perintah yang sama dengan "confirm": true HANYA jika benar-benar diperlukan. SETIAP balasan tepat satu objek JSON: {{"thought","tool","args"}} atau {{"thought","final_answer"}}. Tools: {TOOL_DOCS}""" def parse_json(raw): if not raw: return None t = re.sub(r"```(json)?", "", raw.strip()).strip() try: return json.loads(t) except Exception: pass m = re.search(r"\{[\s\S]*\}", t) try: return json.loads(m.group(0)) if m else None except Exception: return None def render(act_id, history): if len(history) > 10: mid = (f"### CATATAN\n[{len(history) - 10} pesan tengah dipangkas dari prompt. " f"Riwayat UTUH tersedia via history_read('{act_id}', offset, limit)]") body = history[:2] + [{"role": "user", "content": mid}] + history[-8:] else: body = history return "\n\n".join([SYSTEM_PROMPT + f"\n(act_id saat ini: {act_id})"] + [f"### {m['role'].upper()}\n{m['content']}" for m in body]) def audit(event): with _LOCK: log = [] if os.path.exists(AUDIT_PATH): try: log = json.load(open(AUDIT_PATH)) except Exception: log = [] log.append({"ts": time.time(), **event}) with open(AUDIT_PATH, "w") as f: json.dump(log[-500:], f, indent=1) # ================= AGENT LOOP ================= def run_agent(goal, llm, max_steps=12, critic_rounds=1): act = ActLog(f"act-{int(time.time())}") act.write({"event": "goal", "goal": goal}) history = [{"role": "user", "content": goal}] trace = [] try: for step in range(max_steps): raw = llm(render(act.id, history), 1024, 0.2) act.write({"event": "llm", "step": step, "raw": raw}) obj = parse_json(raw) if not obj: history += [{"role": "assistant", "content": raw}, {"role": "user", "content": "ERROR: balasan harus satu objek JSON valid."}] continue if "final_answer" in obj: if critic_rounds > 0: c = parse_json(llm(render(act.id, history + [{"role": "user", "content": f"CRITIC: nilai final_answer terhadap goal. Balas JSON {{\"ok\":bool,\"issues\":\"...\"}}. " f"answer={obj['final_answer'][:2000]}"}]), 512, 0.0)) act.write({"event": "critic", "step": step, "verdict": c}) if c and not c.get("ok", True): history.append({"role": "user", "content": "CRITIC issues: " + str(c.get("issues")) + " Lanjutkan bekerja."}) critic_rounds -= 1 continue return {"answer": obj["final_answer"], "trace": trace, "act_id": act.id} tool = obj.get("tool") args = obj.get("args", {}) or {} note = oversight_note(tool, args) if note and not obj.get("confirm", False): obs = f"RISK_NOTE: {note} Jika setelah berpikir ulang tetap perlu, kirim ulang dengan \"confirm\": true." elif tool not in ALL_TOOLS(): obs = f"ERROR: tool '{tool}' tidak ada." else: try: obs = json.dumps(ALL_TOOLS()[tool](**args), default=str)[:4000] except Exception as e: obs = f"ERROR: {type(e).__name__}: {e}" act.write({"event": "tool", "step": step, "tool": tool, "args": args, "obs": obs}) audit({"event": "tool", "tool": tool, "obs": obs[:200]}) trace.append({"step": step, "tool": tool, "obs": obs[:200]}) history += [{"role": "assistant", "content": raw}, {"role": "user", "content": "OBSERVATION: " + obs}] return {"answer": "max_steps tercapai tanpa final_answer", "trace": trace, "act_id": act.id} finally: fin = history_finalize(act.id) audit({"event": "finalize", "act_id": act.id, **fin})