quess / agent_core.py
SatyaDev13's picture
Upload 3 files
076cd11 verified
Raw History Blame Contribute Delete
17.4 kB
import os, re, json, time, sqlite3, subprocess, importlib.util, base64, socket, ipaddress, threading, shutil
from urllib.parse import urlparse
import requests
from bs4 import BeautifulSoup
WORKSPACE = os.environ.get("AGENT_WORKSPACE", "/tmp/agent_ws")
LOG_DIR = os.path.join(WORKSPACE, "logs")
BACKUP_DIR = os.path.join(WORKSPACE, "backups")
TOOLS_CUSTOM_DIR = os.path.join(WORKSPACE, "tools_custom")
for d in (WORKSPACE, LOG_DIR, BACKUP_DIR, TOOLS_CUSTOM_DIR):
os.makedirs(d, exist_ok=True)
DB_PATH = os.path.join(WORKSPACE, "knowledge.db")
AUDIT_PATH = os.path.join(WORKSPACE, "audit.json")
INDEX_PATH = os.path.join(LOG_DIR, "index.json")
REPO_DIR = os.environ.get("REPO_DIR", "/app")
SPACE_ID = os.environ.get("SPACE_ID", "satyadev13/quess")
GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "")
HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_HUB_TOKEN", "")
UA = {"User-Agent": "Mozilla/5.0 (private-research-agent)"}
_LOCK = threading.Lock()
# ================= REDAKSI SECRET (pra-upload log) =================
def _collect_secrets():
vals = set()
for k in ("GITHUB_TOKEN", "HF_TOKEN", "HUGGINGFACE_HUB_TOKEN", "ADMIN_PASS", "ADMIN_USER"):
v = os.environ.get(k)
if v and len(v) > 5:
vals.add(v)
try:
db = json.load(open(os.path.join(REPO_DIR, "database.json")))
for k in db.get("api_keys", {}):
vals.add(k)
except Exception:
pass
return sorted(vals, key=len, reverse=True)
SECRETS = _collect_secrets()
def redact(t):
for s in SECRETS:
t = t.replace(s, "[REDACTED]")
return t
# ================= ACT LOG: riwayat UTUH per request =================
class ActLog:
def __init__(self, act_id):
self.id = act_id
self.path = os.path.join(LOG_DIR, f"{act_id}.jsonl")
def write(self, event):
with _LOCK:
with open(self.path, "a") as f:
f.write(json.dumps(event, default=str) + "\n")
def read(self, offset=0, limit=50):
if not os.path.exists(self.path):
return []
with open(self.path) as f:
lines = f.readlines()[offset:offset + limit]
return [json.loads(l) for l in lines]
def history_list():
idx = json.load(open(INDEX_PATH)) if os.path.exists(INDEX_PATH) else {}
return {"archived": idx, "local": [f[:-6] for f in os.listdir(LOG_DIR) if f.endswith(".jsonl")]}
def history_read(act_id, offset=0, limit=50):
return ActLog(act_id).read(offset, limit)
def history_summarize(act_id, summary):
idx = json.load(open(INDEX_PATH)) if os.path.exists(INDEX_PATH) else {}
idx.setdefault(act_id, {})["summary"] = summary
json.dump(idx, open(INDEX_PATH, "w"), indent=1)
return {"status": "summary saved", "act_id": act_id}
def history_fetch(url, max_chars=20000):
r = requests.get(url, headers=UA, timeout=60)
r.raise_for_status()
return {"content": r.text[:max_chars]}
def _catbox_upload(path):
with open(path, "rb") as f:
r = requests.post("https://catbox.moe/user/api.php",
data={"reqtype": "fileupload"},
files={"fileToUpload": (os.path.basename(path), f)}, timeout=120)
u = r.text.strip()
return u if u.startswith("http") else None
def history_finalize(act_id):
log = ActLog(act_id)
if not os.path.exists(log.path):
return {"status": "empty"}
tmp = log.path + ".redacted"
with open(log.path) as f:
open(tmp, "w").write(redact(f.read()))
url = _catbox_upload(tmp)
idx = json.load(open(INDEX_PATH)) if os.path.exists(INDEX_PATH) else {}
if url:
os.remove(log.path)
os.remove(tmp)
idx[act_id] = {"url": url, "ts": time.time()}
json.dump(idx, open(INDEX_PATH, "w"), indent=1)
return {"status": "archived", "url": url}
os.remove(tmp)
return {"status": "upload gagal, log lokal dipertahankan", "path": log.path}
# ================= MEMORY (SQLite FTS5) =================
def _db():
con = sqlite3.connect(DB_PATH)
con.execute("CREATE VIRTUAL TABLE IF NOT EXISTS knowledge USING fts5(title, content, source, ts)")
return con
def mem_save(title, content, source=""):
with _LOCK:
con = _db()
con.execute("INSERT INTO knowledge(title,content,source,ts) VALUES(?,?,?,?)",
(title, content, source, time.strftime("%Y-%m-%d %H:%M:%S")))
con.commit()
con.close()
return {"status": "saved"}
def mem_recall(query, limit=5):
con = _db()
try:
rows = con.execute(
"SELECT title, snippet(knowledge,1,'[',']','...',32), source FROM knowledge "
"WHERE knowledge MATCH ? ORDER BY rank LIMIT ?", (query, limit)).fetchall()
except Exception:
like = f"%{query}%"
rows = con.execute(
"SELECT title, substr(content,1,200), source FROM knowledge "
"WHERE title LIKE ? OR content LIKE ? LIMIT ?", (like, like, limit)).fetchall()
con.close()
return [{"title": r[0], "snippet": r[1], "source": r[2]} for r in rows]
# ================= WEB =================
def _safe_url(url):
u = urlparse(url)
if u.scheme not in ("http", "https") or not u.hostname:
return False
try:
ip = ipaddress.ip_address(socket.gethostbyname(u.hostname))
return not (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved)
except Exception:
return False
def web_search(query, max_results=5):
try:
r = requests.post("https://html.duckduckgo.com/html/", data={"q": query}, headers=UA, timeout=20)
soup = BeautifulSoup(r.text, "html.parser")
out = [{"title": a.get_text(strip=True), "url": a.get("href", "")}
for a in soup.select("a.result__a")[:max_results]]
return out or [{"error": "no results"}]
except Exception as e:
return [{"error": str(e)}]
def web_scrape(url, max_chars=8000):
if not _safe_url(url):
return {"error": "URL diblok policy SSRF"}
r = requests.get(url, headers=UA, timeout=30)
r.raise_for_status()
if "html" not in r.headers.get("content-type", ""):
return {"content": r.text[:max_chars]}
soup = BeautifulSoup(r.text, "html.parser")
for t in soup(["script", "style", "nav", "footer", "header"]):
t.decompose()
return {"content": soup.get_text("\n", strip=True)[:max_chars]}
# ================= CODE EXEC (BEBAS: fs + env + jaringan penuh) =================
def code_exec(code, timeout=120, install=""):
if install:
subprocess.run(["pip", "install", "--quiet", install], timeout=300,
capture_output=True, cwd=WORKSPACE)
try:
p = subprocess.run(["python", "-c", code], cwd=WORKSPACE,
capture_output=True, text=True, timeout=timeout)
return {"stdout": p.stdout[-6000:], "stderr": p.stderr[-3000:], "rc": p.returncode}
except subprocess.TimeoutExpired:
return {"error": f"timeout {timeout}s", "rc": -1}
# ================= GITHUB =================
def _gh(method, path, **kw):
r = requests.request(method, f"https://api.github.com{path}", timeout=30,
headers={"Authorization": f"token {GITHUB_TOKEN}",
"Accept": "application/vnd.github+json"}, **kw)
try:
body = r.json()
except Exception:
body = r.text[:500]
return {"status": r.status_code, "body": body}
def gh_list_repos(): return _gh("GET", "/user/repos?per_page=100")
def gh_search(query): return _gh("GET", f"/search/repositories?q={requests.utils.quote(query)}")
def gh_create_repo(name, private=True): return _gh("POST", "/user/repos", json={"name": name, "private": private})
def gh_push_file(repo, path, content, message="agent commit"):
cur = _gh("GET", f"/repos/{repo}/contents/{path}")
payload = {"message": message, "content": base64.b64encode(content.encode()).decode()}
if cur["status"] == 200 and isinstance(cur["body"], dict) and cur["body"].get("sha"):
payload["sha"] = cur["body"]["sha"]
return _gh("PUT", f"/repos/{repo}/contents/{path}", json=payload)
def gh_create_action(repo, name, yaml_content):
return gh_push_file(repo, f".github/workflows/{name}.yml", yaml_content, f"agent: workflow {name}")
def gh_delete_repo(repo): return _gh("DELETE", f"/repos/{repo}")
# ================= MODIFIKASI DIRI (bebas + snapshot otomatis) =================
CUSTOM_TOOLS = {}
def load_custom_tools():
CUSTOM_TOOLS.clear()
for fn in os.listdir(TOOLS_CUSTOM_DIR):
if not fn.endswith(".py"):
continue
name = fn[:-3]
spec = importlib.util.spec_from_file_location(name, os.path.join(TOOLS_CUSTOM_DIR, fn))
mod = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(mod)
for attr in dir(mod):
o = getattr(mod, attr)
if callable(o) and getattr(o, "__module__", "") == mod.__name__ and not attr.startswith("_"):
CUSTOM_TOOLS["custom_" + attr] = o
except Exception as e:
CUSTOM_TOOLS[f"custom_{name}_LOAD_ERROR"] = lambda *a, e=str(e), **k: {"error": e}
load_custom_tools()
def _repo_path(rel):
return os.path.normpath(os.path.join(REPO_DIR, rel))
def self_list(rel=""):
p = _repo_path(rel) if rel else REPO_DIR
if not os.path.isdir(p):
return {"error": "not a dir"}
out = []
for root, dirs, files in os.walk(p):
dirs[:] = [d for d in dirs if not d.startswith(".")]
for f in files:
out.append(os.path.relpath(os.path.join(root, f), REPO_DIR))
return {"files": out[:300]}
def self_read(rel, max_chars=20000):
p = _repo_path(rel)
if not os.path.isfile(p):
return {"error": "not found"}
return {"content": open(p, errors="replace").read()[:max_chars]}
def self_edit(rel, content):
p = _repo_path(rel)
os.makedirs(os.path.dirname(p) or ".", exist_ok=True)
if os.path.exists(p):
shutil.copy2(p, os.path.join(BACKUP_DIR, os.path.basename(p) + f".{int(time.time())}.bak"))
with open(p, "w") as f:
f.write(content)
return {"status": "written", "path": p,
"note": "perlu self_restart agar berlaku untuk app.py/agent_core.py"}
def self_rollback(rel):
p = _repo_path(rel)
b = os.path.basename(p)
c = sorted([f for f in os.listdir(BACKUP_DIR) if f.startswith(b + ".")])
if not c:
return {"error": "no backup"}
shutil.copy2(os.path.join(BACKUP_DIR, c[-1]), p)
return {"status": "rolled back", "from": c[-1]}
def self_restart():
r = requests.post(f"https://huggingface.co/api/spaces/{SPACE_ID}/restart",
headers={"Authorization": f"Bearer {HF_TOKEN}"}, timeout=30)
return {"status": r.status_code}
# ================= OVERSIGHT ADVISORY (think-twice, tanpa gerbang manusia) =================
def oversight_note(tool, args):
if tool == "gh_delete_repo":
return "DESTRUKTIF IRREVERSIBEL: repo hilang permanen dari GitHub."
if tool == "self_edit":
return "MODIFIKASI DIRI: snapshot backup dibuat otomatis; self_rollback tersedia."
if tool == "self_restart":
return "Restart memutus SEMUA job yang sedang berjalan."
if tool == "code_exec" and re.search(r"rm\s+-rf\s+/|shutil\.rmtree\(['\"]/|mkfs", str(args.get("code", ""))):
return "POLA MERUSAK SISTEM: kontainer bisa mati total dan Space rebuild."
return None
# ================= REGISTRY =================
BASE_TOOLS = {
"web_search": web_search, "web_scrape": web_scrape, "code_exec": code_exec,
"mem_save": mem_save, "mem_recall": mem_recall,
"history_list": history_list, "history_read": history_read,
"history_summarize": history_summarize, "history_fetch": history_fetch,
"gh_list_repos": gh_list_repos, "gh_search": gh_search, "gh_create_repo": gh_create_repo,
"gh_push_file": gh_push_file, "gh_create_action": gh_create_action, "gh_delete_repo": gh_delete_repo,
"self_list": self_list, "self_read": self_read, "self_edit": self_edit,
"self_rollback": self_rollback, "self_restart": self_restart,
}
def ALL_TOOLS():
return {**BASE_TOOLS, **CUSTOM_TOOLS}
TOOL_DOCS = ("web_search(query)|web_scrape(url)|code_exec(code,timeout,install) bebas baca fs/env| "
"mem_save(title,content,source)|mem_recall(query)| "
"history_list()|history_read(act_id,offset,limit)|history_summarize(act_id,summary)|history_fetch(url)| "
"gh_list_repos()|gh_search(query)|gh_create_repo(name)|gh_push_file(repo,path,content,message)|"
"gh_create_action(repo,name,yaml)|gh_delete_repo(repo)| "
"self_list(rel)|self_read(rel)|self_edit(rel,content)|self_rollback(rel)|self_restart()| custom_*")
SYSTEM_PROMPT = f"""Kamu adalah Quess-Agent v2, agen riset otonom tanpa pengawas manusia. Kejar tujuan sampai tuntas: cari, scrape, eksekusi kode, belajar, simpan, ulangi. Jangan menyerah pada kegagalan tool pertama; coba jalur lain.
Riwayat langkahmu saat ini tersimpan UTUH di act log; baca potongan mana pun via history_read. Ringkas via history_summarize tanpa menghapus asli.
Jika menerima RISK_NOTE, wajib berpikir ulang secara matang: kirim ulang perintah yang sama dengan "confirm": true HANYA jika benar-benar diperlukan.
SETIAP balasan tepat satu objek JSON: {{"thought","tool","args"}} atau {{"thought","final_answer"}}.
Tools: {TOOL_DOCS}"""
def parse_json(raw):
if not raw:
return None
t = re.sub(r"```(json)?", "", raw.strip()).strip()
try:
return json.loads(t)
except Exception:
pass
m = re.search(r"\{[\s\S]*\}", t)
try:
return json.loads(m.group(0)) if m else None
except Exception:
return None
def render(act_id, history):
if len(history) > 10:
mid = (f"### CATATAN\n[{len(history) - 10} pesan tengah dipangkas dari prompt. "
f"Riwayat UTUH tersedia via history_read('{act_id}', offset, limit)]")
body = history[:2] + [{"role": "user", "content": mid}] + history[-8:]
else:
body = history
return "\n\n".join([SYSTEM_PROMPT + f"\n(act_id saat ini: {act_id})"] +
[f"### {m['role'].upper()}\n{m['content']}" for m in body])
def audit(event):
with _LOCK:
log = []
if os.path.exists(AUDIT_PATH):
try:
log = json.load(open(AUDIT_PATH))
except Exception:
log = []
log.append({"ts": time.time(), **event})
with open(AUDIT_PATH, "w") as f:
json.dump(log[-500:], f, indent=1)
# ================= AGENT LOOP =================
def run_agent(goal, llm, max_steps=12, critic_rounds=1):
act = ActLog(f"act-{int(time.time())}")
act.write({"event": "goal", "goal": goal})
history = [{"role": "user", "content": goal}]
trace = []
try:
for step in range(max_steps):
raw = llm(render(act.id, history), 1024, 0.2)
act.write({"event": "llm", "step": step, "raw": raw})
obj = parse_json(raw)
if not obj:
history += [{"role": "assistant", "content": raw},
{"role": "user", "content": "ERROR: balasan harus satu objek JSON valid."}]
continue
if "final_answer" in obj:
if critic_rounds > 0:
c = parse_json(llm(render(act.id, history + [{"role": "user", "content":
f"CRITIC: nilai final_answer terhadap goal. Balas JSON {{\"ok\":bool,\"issues\":\"...\"}}. "
f"answer={obj['final_answer'][:2000]}"}]), 512, 0.0))
act.write({"event": "critic", "step": step, "verdict": c})
if c and not c.get("ok", True):
history.append({"role": "user",
"content": "CRITIC issues: " + str(c.get("issues")) + " Lanjutkan bekerja."})
critic_rounds -= 1
continue
return {"answer": obj["final_answer"], "trace": trace, "act_id": act.id}
tool = obj.get("tool")
args = obj.get("args", {}) or {}
note = oversight_note(tool, args)
if note and not obj.get("confirm", False):
obs = f"RISK_NOTE: {note} Jika setelah berpikir ulang tetap perlu, kirim ulang dengan \"confirm\": true."
elif tool not in ALL_TOOLS():
obs = f"ERROR: tool '{tool}' tidak ada."
else:
try:
obs = json.dumps(ALL_TOOLS()[tool](**args), default=str)[:4000]
except Exception as e:
obs = f"ERROR: {type(e).__name__}: {e}"
act.write({"event": "tool", "step": step, "tool": tool, "args": args, "obs": obs})
audit({"event": "tool", "tool": tool, "obs": obs[:200]})
trace.append({"step": step, "tool": tool, "obs": obs[:200]})
history += [{"role": "assistant", "content": raw},
{"role": "user", "content": "OBSERVATION: " + obs}]
return {"answer": "max_steps tercapai tanpa final_answer", "trace": trace, "act_id": act.id}
finally:
fin = history_finalize(act.id)
audit({"event": "finalize", "act_id": act.id, **fin})