fix: bound the waits that could freeze the learn loop (voyager_stack.py)

voyager_stack.py

@@ -793,6 +793,153 @@ async function exploreUntil(bot, direction, maxTime = 60, callback) {
             action_text = action_text.replace(marker, creative_facts + "\n" + marker, 1)
             action_prompt.write_text(action_text, encoding="utf-8")
 
+        # SkillManager.__init__ built Chroma() with no deadline at all, and
+        # Chroma embeds through OpenAIEmbeddings(), which in langchain 0.0.350
+        # calls openai==0.28.1 with no request timeout. A single stalled
+        # /v1/embeddings response therefore parked the constructor forever:
+        # the log stopped at "Loading Skill Manager" and the loop never reached
+        # learn(), so nothing failed, nothing retried and no counter moved.
+        # The stall watchdog then fired every 900s and looked like the cause
+        # instead of the symptom.
+        #
+        # The same constructor ended with a hard assert. A vectordb that is
+        # one document out of sync is recoverable -- the skills.json on disk is
+        # the source of truth -- but the assert turned that into a dead process.
+        # Rebuild the index from skills.json instead, which is what the comment
+        # in the failure message already told the operator to do by hand.
+        skill_path = SRC / "voyager" / "agents" / "skill.py"
+        skill_text = skill_path.read_text(encoding="utf-8")
+        old_vectordb = '''        self.vectordb = Chroma(
+            collection_name="skill_vectordb",
+            embedding_function=OpenAIEmbeddings(),
+            persist_directory=f"{ckpt_dir}/skill/vectordb",
+        )'''
+        new_vectordb = '''        self.vectordb = self._open_vectordb(ckpt_dir)'''
+        if old_vectordb in skill_text:
+            skill_text = skill_text.replace(old_vectordb, new_vectordb, 1)
+        elif new_vectordb.strip() not in skill_text:
+            # The anchor is gone and the replacement is not already in place:
+            # the file changed underneath this patch and the unbounded
+            # constructor would survive. Refuse rather than boot a tree that
+            # can still hang.
+            raise RuntimeError("SkillManager patch anchor missing; refusing unbounded-Chroma patch")
+
+        old_assert = '''        assert self.vectordb._collection.count() == len(self.skills), (
+            f"Skill Manager's vectordb is not synced with skills.json.\\n"
+            f"There are {self.vectordb._collection.count()} skills in vectordb but {len(self.skills)} skills in skills.json.\\n"
+            f"Did you set resume=False when initializing the manager?\\n"
+            f"You may need to manually delete the vectordb directory for running from scratch."
+        )'''
+        new_resync = '''        self._resync_vectordb()'''
+        if old_assert in skill_text:
+            skill_text = skill_text.replace(old_assert, new_resync, 1)
+        elif "self._resync_vectordb()" not in skill_text:
+            raise RuntimeError("SkillManager sync-assert anchor missing; refusing patch")
+
+        skill_helpers = '''    # Every embedding this class makes goes through here, so one stalled
+    # response cannot freeze the constructor, add_new_skill or retrieve_skills.
+    EMBED_DEADLINE_SECONDS = 60
+    VECTORDB_OPEN_SECONDS = 180
+
+    def _embedding_function(self):
+        """OpenAIEmbeddings with an explicit timeout.
+
+        langchain 0.0.350 builds the openai 0.28.1 client, whose
+        Embedding.create has no default request timeout. The base URL is the
+        remote brain Space, so a slow or wedged response is a real possibility
+        and must surface as an exception the runner can retry, not as a hang.
+
+        request_timeout is a declared pydantic field in this version, so pass it
+        to the constructor. Some versions renamed it to timeout, and a pydantic
+        model raises on an unknown keyword, so fall back rather than crash the
+        constructor over an observability nicety.
+        """
+        try:
+            return OpenAIEmbeddings(request_timeout=self.EMBED_DEADLINE_SECONDS)
+        except Exception:
+            try:
+                return OpenAIEmbeddings(timeout=self.EMBED_DEADLINE_SECONDS)
+            except Exception:
+                return OpenAIEmbeddings()
+
+    def _open_vectordb(self, ckpt_dir):
+        """Construct Chroma under a wall-clock deadline.
+
+        Chroma can touch the embedding function while opening (and will, when
+        it has to write a missing index). Running the open in a worker gives
+        the constructor a deadline it did not have; the caller follows up with
+        _resync_vectordb, which repairs any index that is out of date.
+        """
+        import concurrent.futures as _cf
+
+        def _open():
+            return Chroma(
+                collection_name="skill_vectordb",
+                embedding_function=self._embedding_function(),
+                persist_directory=f"{ckpt_dir}/skill/vectordb",
+            )
+
+        with _cf.ThreadPoolExecutor(max_workers=1) as pool:
+            future = pool.submit(_open)
+            try:
+                return future.result(timeout=self.VECTORDB_OPEN_SECONDS)
+            except _cf.TimeoutError as exc:
+                raise RuntimeError(
+                    "vector store did not open within "
+                    f"{self.VECTORDB_OPEN_SECONDS}s; the embedding endpoint is "
+                    "not answering"
+                ) from exc
+
+    def _resync_vectordb(self):
+        """Make the index match skills.json, which is the source of truth.
+
+        Replaces a hard assert that killed the process when the two counts
+        disagreed by one document. Rebuilding is safe: every skill is present
+        in skills.json with its code and description.
+        """
+        expected = set(self.skills)
+        try:
+            stored = set(self.vectordb._collection.get()["ids"])
+        except Exception:
+            stored = set()
+        for stale in stored - expected:
+            try:
+                self.vectordb._collection.delete(ids=[stale])
+            except Exception:
+                pass
+        missing = expected - stored
+        if missing:
+            print(
+                f"\\033[33mSkill Manager resyncing {len(missing)} skill(s) into the vectordb\\033[0m",
+                flush=True,
+            )
+            for name in sorted(missing):
+                entry = self.skills.get(name) or {}
+                try:
+                    self.vectordb.add_texts(
+                        texts=[f"Skill: {name}\\n{entry.get('description', '')}"],
+                        metadatas=[{"name": name}],
+                        ids=[name],
+                    )
+                except Exception as exc:
+                    print(
+                        f"\\033[33mSkill Manager could not index {name}: "
+                        f"{type(exc).__name__}: {exc}\\033[0m",
+                        flush=True,
+                    )
+        try:
+            self.vectordb.persist()
+        except Exception:
+            pass
+
+'''
+        if "def _open_vectordb" not in skill_text:
+            marker_after = "    @property\n    def programs(self):"
+            if marker_after not in skill_text:
+                raise RuntimeError("SkillManager property anchor missing; refusing patch")
+            skill_text = skill_text.replace(marker_after, skill_helpers + marker_after, 1)
+        skill_path.write_text(skill_text, encoding="utf-8")
+
         action_agent_path = SRC / "voyager" / "agents" / "action.py"
         action_agent_text = action_agent_path.read_text(encoding="utf-8")
         action_agent_text = action_agent_text.replace(