init AI doc extractor

.gitignore

@@ -0,0 +1,18 @@
+# Deno
+.env
+deno.lock
+
+# Node modules
+node_modules/
+
+# Generated docs
+generated-docs/*.md
+
+# Frontend build
+frontend/dist/
+frontend/node_modules/
+
+# OS
+.DS_Store
+Thumbs.db
+

README.md

@@ -0,0 +1,66 @@
+# Docs Extractor - AI-Powered Documentation Generator
+
+An intelligent documentation extraction tool powered by Zypher Agent and Firecrawl. This tool automatically crawls documentation websites and generates comprehensive guides based on your needs.
+
+
+## Setup
+
+### Prerequisites
+
+1. [Deno](https://deno.com) installed
+2. [Node.js and npm](https://nodejs.org) installed (for frontend)
+3. API Keys:
+   - Anthropic API key (for Claude AI)
+   - Firecrawl API key (for web crawling)
+
+### Installation
+
+1. **Set up environment variables:**
+
+   Create a `.env` file in the `docs-extractor` directory:
+
+   ```bash
+   ANTHROPIC_API_KEY=your_anthropic_api_key_here
+   FIRECRAWL_API_KEY=your_firecrawl_api_key_here
+   ```
+
+   Get your API keys:
+   - Anthropic: https://console.anthropic.com/
+   - Firecrawl: https://firecrawl.dev/
+
+2. **Install frontend dependencies:**
+
+   ```bash
+   cd frontend
+   npm install
+   ```
+
+## Usage
+
+### Option 1: With UI (Recommended)
+
+1. **Start the backend server:**
+
+   ```bash
+   deno task server
+   ```
+
+   The API server will start on `http://localhost:8000`
+
+2. **Start the frontend (in a new terminal):**
+
+   ```bash
+   cd frontend
+   npm run dev
+   ```
+
+   The UI will be available at `http://localhost:5173`
+
+3. **Use the web interface:**
+   - Enter a documentation URL (e.g., `https://react.dev/docs`)
+   - Specify what you want to extract (e.g., "Quickstart guide with installation and examples")
+   - Click "Extract Documentation"
+   - Wait for the AI to process and generate the documentation
+   - View and download the generated markdown file
+
+

deno.json

@@ -0,0 +1,12 @@
+{
+  "tasks": {
+    "dev": "deno run -A main.ts",
+    "server": "deno run -A server.ts"
+  },
+  "imports": {
+    "@std/dotenv": "jsr:@std/dotenv@^0.225.0",
+    "@corespeed/zypher": "jsr:@corespeed/zypher@^0.4.0",
+    "@oak/oak": "jsr:@oak/oak@^17.1.3",
+    "rxjs-for-await": "npm:rxjs-for-await@^1.0.0"
+  }
+}

frontend/index.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Docs Extractor - AI-Powered Documentation Generator</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
+

frontend/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "docs-extractor-frontend",
+  "private": true,
+  "version": "0.1.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "react-markdown": "^9.0.1",
+    "remark-gfm": "^4.0.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.12",
+    "@types/react-dom": "^18.3.1",
+    "@vitejs/plugin-react": "^4.3.4",
+    "typescript": "^5.7.2",
+    "vite": "^6.0.1"
+  }
+}
+

frontend/src/App.css

@@ -0,0 +1,15 @@
+.app {
+  min-height: 100vh;
+  display: flex;
+  flex-direction: column;
+}
+
+.form-section {
+  display: flex;
+  flex-direction: column;
+}
+
+.output-section {
+  flex: 1;
+}
+

frontend/src/App.tsx

@@ -0,0 +1,296 @@
+import { useState, useEffect } from 'react';
+import ReactMarkdown from 'react-markdown';
+import remarkGfm from 'remark-gfm';
+import './App.css';
+
+interface Job {
+  status: 'processing' | 'completed' | 'failed';
+  result?: {
+    response: {
+      content: Array<{ type: string; text: string }>;
+    };
+  };
+  error?: string;
+}
+
+interface Doc {
+  name: string;
+  path: string;
+}
+
+function App() {
+  const [docsUrl, setDocsUrl] = useState('');
+  const [extractionType, setExtractionType] = useState('');
+  const [loading, setLoading] = useState(false);
+  const [jobId, setJobId] = useState<string | null>(null);
+  const [result, setResult] = useState<string | null>(null);
+  const [error, setError] = useState<string | null>(null);
+  const [generatedDocs, setGeneratedDocs] = useState<Doc[]>([]);
+  const [selectedDoc, setSelectedDoc] = useState<string | null>(null);
+  const [docContent, setDocContent] = useState<string | null>(null);
+
+  const API_BASE = 'http://localhost:8000';
+
+  const loadGeneratedDocs = async () => {
+    try {
+      const response = await fetch(`${API_BASE}/api/docs`);
+      const data = await response.json();
+      setGeneratedDocs(data.docs || []);
+    } catch (err) {
+      console.error('Error loading docs:', err);
+    }
+  };
+
+  const loadDocContent = async (filename: string) => {
+    try {
+      const response = await fetch(`${API_BASE}/api/docs/${filename}`);
+      const content = await response.text();
+      setDocContent(content);
+      setSelectedDoc(filename);
+    } catch (err) {
+      console.error('Error loading doc content:', err);
+    }
+  };
+
+  useEffect(() => {
+    loadGeneratedDocs();
+  }, []);
+
+  useEffect(() => {
+    if (!jobId) return;
+
+    const pollInterval = setInterval(async () => {
+      try {
+        const response = await fetch(`${API_BASE}/api/job/${jobId}`);
+        const job: Job = await response.json();
+
+        if (job.status === 'completed') {
+          clearInterval(pollInterval);
+          setLoading(false);
+          
+          if (job.result?.response?.content) {
+            const textContent = job.result.response.content
+              .filter(item => item.type === 'text')
+              .map(item => item.text)
+              .join('\n\n');
+            setResult(textContent);
+          }
+          
+          setJobId(null);
+          loadGeneratedDocs();
+        } else if (job.status === 'failed') {
+          clearInterval(pollInterval);
+          setLoading(false);
+          setError(job.error || 'Extraction failed');
+          setJobId(null);
+        }
+      } catch (err) {
+        console.error('Error polling job:', err);
+      }
+    }, 2000);
+
+    return () => clearInterval(pollInterval);
+  }, [jobId]);
+
+  const handleSubmit = async (e: React.FormEvent) => {
+    e.preventDefault();
+    setLoading(true);
+    setError(null);
+    setResult(null);
+
+    try {
+      const response = await fetch(`${API_BASE}/api/extract`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ docsUrl, extractionType }),
+      });
+
+      if (!response.ok) {
+        throw new Error('Failed to start extraction');
+      }
+
+      const data = await response.json();
+      setJobId(data.jobId);
+    } catch (err) {
+      setLoading(false);
+      setError(err instanceof Error ? err.message : 'An error occurred');
+    }
+  };
+
+  const exampleTypes = [
+    'Quick installation guide',
+    'Main features overview',
+    'Getting started tutorial',
+    'API reference summary',
+    'Configuration guide',
+  ];
+
+  return (
+    <div className="app">
+      <header className="header">
+        <div className="container">
+          <h1>📚 Docs Extractor</h1>
+          <p className="subtitle">AI-powered documentation generator using Zypher Agent</p>
+        </div>
+      </header>
+
+      <main className="container main-content">
+        <div className="grid">
+          <div className="form-section">
+            <div className="card">
+              <h2>Extract Documentation</h2>
+              <form onSubmit={handleSubmit}>
+                <div className="form-group">
+                  <label htmlFor="docsUrl">
+                    Documentation URL
+                    <span className="required">*</span>
+                  </label>
+                  <input
+                    id="docsUrl"
+                    type="url"
+                    placeholder="https://example.com/docs"
+                    value={docsUrl}
+                    onChange={(e) => setDocsUrl(e.target.value)}
+                    required
+                    disabled={loading}
+                  />
+                  <small>Enter the URL of the documentation you want to extract</small>
+                </div>
+
+                <div className="form-group">
+                  <label htmlFor="extractionType">
+                    What do you want?
+                    <span className="required">*</span>
+                  </label>
+                  <textarea
+                    id="extractionType"
+                    placeholder="e.g., Quickstart guide with installation steps and examples"
+                    value={extractionType}
+                    onChange={(e) => setExtractionType(e.target.value)}
+                    required
+                    disabled={loading}
+                    rows={3}
+                  />
+                  <small>Describe what you want to extract from the documentation</small>
+                </div>
+
+                <div className="example-types">
+                  <strong>Example requests:</strong>
+                  {exampleTypes.map((type, i) => (
+                    <button
+                      key={i}
+                      type="button"
+                      className="example-btn"
+                      onClick={() => setExtractionType(type)}
+                      disabled={loading}
+                    >
+                      {type}
+                    </button>
+                  ))}
+                </div>
+
+                <button type="submit" className="btn btn-primary" disabled={loading}>
+                  {loading ? (
+                    <>
+                      <span className="spinner"></span>
+                      Processing...
+                    </>
+                  ) : (
+                    'Extract Documentation'
+                  )}
+                </button>
+              </form>
+
+              {error && (
+                <div className="alert alert-error">
+                  <strong>Error:</strong> {error}
+                </div>
+              )}
+            </div>
+
+            <div className="card">
+              <h3>Generated Documents</h3>
+              {generatedDocs.length === 0 ? (
+                <p className="empty-state">No documents generated yet</p>
+              ) : (
+                <ul className="docs-list">
+                  {generatedDocs.map((doc) => (
+                    <li key={doc.name}>
+                      <button
+                        className={`doc-btn ${selectedDoc === doc.name ? 'active' : ''}`}
+                        onClick={() => loadDocContent(doc.name)}
+                      >
+                        📄 {doc.name}
+                      </button>
+                    </li>
+                  ))}
+                </ul>
+              )}
+            </div>
+          </div>
+
+          <div className="output-section">
+            <div className="card output-card">
+              <h2>Output</h2>
+              {loading && (
+                <div className="loading-state">
+                  <div className="spinner large"></div>
+                  <p>AI is extracting and generating documentation...</p>
+                  <small>This may take a few minutes depending on the size of the documentation</small>
+                </div>
+              )}
+
+              {result && (
+                <div className="result">
+                  <div className="result-header">
+                    <span className="badge badge-success">✓ Generated</span>
+                  </div>
+                  <div className="markdown-content">
+                    <ReactMarkdown remarkPlugins={[remarkGfm]}>
+                      {result}
+                    </ReactMarkdown>
+                  </div>
+                </div>
+              )}
+
+              {docContent && !result && (
+                <div className="result">
+                  <div className="result-header">
+                    <span className="badge badge-info">📄 {selectedDoc}</span>
+                  </div>
+                  <div className="markdown-content">
+                    <ReactMarkdown remarkPlugins={[remarkGfm]}>
+                      {docContent}
+                    </ReactMarkdown>
+                  </div>
+                </div>
+              )}
+
+              {!loading && !result && !docContent && (
+                <div className="empty-state">
+                  <p>👈 Enter a documentation URL and select what you want to extract</p>
+                  <p className="hint">The AI will analyze the documentation and generate a comprehensive guide for you</p>
+                </div>
+              )}
+            </div>
+          </div>
+        </div>
+      </main>
+
+      <footer className="footer">
+        <div className="container">
+          <p>
+            Powered by <a href="https://zypher.corespeed.io" target="_blank" rel="noopener noreferrer">Zypher Agent</a>
+            {' '}&amp;{' '}
+            <a href="https://firecrawl.dev" target="_blank" rel="noopener noreferrer">Firecrawl</a>
+          </p>
+        </div>
+      </footer>
+    </div>
+  );
+}
+
+export default App;
+

frontend/src/index.css

@@ -0,0 +1,464 @@
+:root {
+  --primary: #6366f1;
+  --primary-dark: #4f46e5;
+  --success: #10b981;
+  --error: #ef4444;
+  --warning: #f59e0b;
+  --bg: #0f172a;
+  --bg-card: #1e293b;
+  --bg-hover: #334155;
+  --text: #f1f5f9;
+  --text-muted: #94a3b8;
+  --border: #334155;
+  --radius: 12px;
+  --shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.3);
+  --shadow-lg: 0 10px 15px -3px rgba(0, 0, 0, 0.4);
+}
+
+* {
+  margin: 0;
+  padding: 0;
+  box-sizing: border-box;
+}
+
+body {
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
+    'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
+    sans-serif;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  background: var(--bg);
+  color: var(--text);
+  line-height: 1.6;
+}
+
+code {
+  font-family: 'Courier New', Courier, monospace;
+}
+
+.container {
+  max-width: 1400px;
+  margin: 0 auto;
+  padding: 0 2rem;
+}
+
+.header {
+  background: var(--bg-card);
+  border-bottom: 1px solid var(--border);
+  padding: 2rem 0;
+  margin-bottom: 2rem;
+}
+
+.header h1 {
+  font-size: 2.5rem;
+  font-weight: 700;
+  margin-bottom: 0.5rem;
+  background: linear-gradient(135deg, var(--primary) 0%, #8b5cf6 100%);
+  -webkit-background-clip: text;
+  -webkit-text-fill-color: transparent;
+  background-clip: text;
+}
+
+.subtitle {
+  color: var(--text-muted);
+  font-size: 1.1rem;
+}
+
+.main-content {
+  padding-bottom: 4rem;
+}
+
+.grid {
+  display: grid;
+  grid-template-columns: 450px 1fr;
+  gap: 2rem;
+  align-items: start;
+}
+
+@media (max-width: 1024px) {
+  .grid {
+    grid-template-columns: 1fr;
+  }
+}
+
+.card {
+  background: var(--bg-card);
+  border: 1px solid var(--border);
+  border-radius: var(--radius);
+  padding: 1.5rem;
+  box-shadow: var(--shadow);
+  margin-bottom: 1.5rem;
+}
+
+.card h2 {
+  font-size: 1.5rem;
+  margin-bottom: 1.5rem;
+  color: var(--text);
+}
+
+.card h3 {
+  font-size: 1.2rem;
+  margin-bottom: 1rem;
+  color: var(--text);
+}
+
+.form-group {
+  margin-bottom: 1.5rem;
+}
+
+.form-group label {
+  display: block;
+  margin-bottom: 0.5rem;
+  font-weight: 500;
+  color: var(--text);
+}
+
+.required {
+  color: var(--error);
+  margin-left: 0.25rem;
+}
+
+.form-group input,
+.form-group textarea {
+  width: 100%;
+  padding: 0.75rem;
+  background: var(--bg);
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  color: var(--text);
+  font-size: 1rem;
+  transition: all 0.2s;
+}
+
+.form-group input:focus,
+.form-group textarea:focus {
+  outline: none;
+  border-color: var(--primary);
+  box-shadow: 0 0 0 3px rgba(99, 102, 241, 0.1);
+}
+
+.form-group input:disabled,
+.form-group textarea:disabled {
+  opacity: 0.6;
+  cursor: not-allowed;
+}
+
+.form-group small {
+  display: block;
+  margin-top: 0.5rem;
+  color: var(--text-muted);
+  font-size: 0.875rem;
+}
+
+.example-types {
+  margin-bottom: 1.5rem;
+  padding: 1rem;
+  background: var(--bg);
+  border-radius: 8px;
+}
+
+.example-types strong {
+  display: block;
+  margin-bottom: 0.75rem;
+  color: var(--text-muted);
+  font-size: 0.875rem;
+}
+
+.example-btn {
+  display: block;
+  width: 100%;
+  text-align: left;
+  padding: 0.5rem 0.75rem;
+  margin-bottom: 0.5rem;
+  background: transparent;
+  border: 1px solid var(--border);
+  border-radius: 6px;
+  color: var(--text);
+  font-size: 0.875rem;
+  cursor: pointer;
+  transition: all 0.2s;
+}
+
+.example-btn:hover:not(:disabled) {
+  background: var(--bg-hover);
+  border-color: var(--primary);
+}
+
+.example-btn:disabled {
+  opacity: 0.5;
+  cursor: not-allowed;
+}
+
+.btn {
+  padding: 0.875rem 1.5rem;
+  border: none;
+  border-radius: 8px;
+  font-size: 1rem;
+  font-weight: 600;
+  cursor: pointer;
+  transition: all 0.2s;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  gap: 0.5rem;
+}
+
+.btn-primary {
+  background: var(--primary);
+  color: white;
+  width: 100%;
+}
+
+.btn-primary:hover:not(:disabled) {
+  background: var(--primary-dark);
+  transform: translateY(-1px);
+  box-shadow: var(--shadow-lg);
+}
+
+.btn-primary:disabled {
+  opacity: 0.6;
+  cursor: not-allowed;
+  transform: none;
+}
+
+.spinner {
+  display: inline-block;
+  width: 16px;
+  height: 16px;
+  border: 2px solid rgba(255, 255, 255, 0.3);
+  border-top-color: white;
+  border-radius: 50%;
+  animation: spin 0.8s linear infinite;
+}
+
+.spinner.large {
+  width: 48px;
+  height: 48px;
+  border-width: 4px;
+}
+
+@keyframes spin {
+  to { transform: rotate(360deg); }
+}
+
+.alert {
+  padding: 1rem;
+  border-radius: 8px;
+  margin-top: 1rem;
+}
+
+.alert-error {
+  background: rgba(239, 68, 68, 0.1);
+  border: 1px solid var(--error);
+  color: #fca5a5;
+}
+
+.docs-list {
+  list-style: none;
+}
+
+.docs-list li {
+  margin-bottom: 0.5rem;
+}
+
+.doc-btn {
+  width: 100%;
+  text-align: left;
+  padding: 0.75rem 1rem;
+  background: var(--bg);
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  color: var(--text);
+  cursor: pointer;
+  transition: all 0.2s;
+}
+
+.doc-btn:hover {
+  background: var(--bg-hover);
+  border-color: var(--primary);
+}
+
+.doc-btn.active {
+  background: var(--primary);
+  border-color: var(--primary);
+  color: white;
+}
+
+.output-card {
+  min-height: 600px;
+  position: sticky;
+  top: 2rem;
+  overflow: hidden;
+}
+
+.output-card .markdown-content {
+  max-width: 100%;
+  overflow-x: hidden;
+}
+
+.loading-state {
+  text-align: center;
+  padding: 4rem 2rem;
+}
+
+.loading-state p {
+  margin-top: 1rem;
+  font-size: 1.1rem;
+}
+
+.loading-state small {
+  display: block;
+  margin-top: 0.5rem;
+  color: var(--text-muted);
+}
+
+.result-header {
+  margin-bottom: 1.5rem;
+  padding-bottom: 1rem;
+  border-bottom: 1px solid var(--border);
+}
+
+.badge {
+  display: inline-flex;
+  align-items: center;
+  padding: 0.375rem 0.75rem;
+  border-radius: 6px;
+  font-size: 0.875rem;
+  font-weight: 600;
+}
+
+.badge-success {
+  background: rgba(16, 185, 129, 0.1);
+  color: var(--success);
+}
+
+.badge-info {
+  background: rgba(99, 102, 241, 0.1);
+  color: var(--primary);
+}
+
+.empty-state {
+  text-align: center;
+  padding: 4rem 2rem;
+  color: var(--text-muted);
+}
+
+.empty-state p {
+  margin-bottom: 0.5rem;
+}
+
+.hint {
+  font-size: 0.875rem;
+}
+
+.markdown-content {
+  color: var(--text);
+  line-height: 1.8;
+  word-wrap: break-word;
+  overflow-wrap: break-word;
+  max-width: 100%;
+}
+
+.markdown-content h1,
+.markdown-content h2,
+.markdown-content h3,
+.markdown-content h4 {
+  margin-top: 2rem;
+  margin-bottom: 1rem;
+  color: var(--text);
+  word-wrap: break-word;
+}
+
+.markdown-content h1 { font-size: 2rem; }
+.markdown-content h2 { font-size: 1.5rem; }
+.markdown-content h3 { font-size: 1.25rem; }
+
+.markdown-content p {
+  margin-bottom: 1rem;
+  word-wrap: break-word;
+  overflow-wrap: break-word;
+}
+
+.markdown-content a {
+  color: var(--primary);
+  text-decoration: none;
+  word-wrap: break-word;
+  overflow-wrap: break-word;
+}
+
+.markdown-content a:hover {
+  text-decoration: underline;
+}
+
+.markdown-content code {
+  background: var(--bg);
+  padding: 0.2rem 0.4rem;
+  border-radius: 4px;
+  font-size: 0.875em;
+  word-wrap: break-word;
+  white-space: pre-wrap;
+}
+
+.markdown-content pre {
+  background: var(--bg);
+  padding: 1rem;
+  border-radius: 8px;
+  overflow-x: auto;
+  margin: 1rem 0;
+  max-width: 100%;
+}
+
+.markdown-content pre code {
+  background: none;
+  padding: 0;
+  white-space: pre;
+  word-wrap: normal;
+}
+
+.markdown-content ul,
+.markdown-content ol {
+  margin-left: 1.5rem;
+  margin-bottom: 1rem;
+}
+
+.markdown-content blockquote {
+  border-left: 4px solid var(--primary);
+  padding-left: 1rem;
+  margin: 1rem 0;
+  color: var(--text-muted);
+}
+
+.markdown-content table {
+  width: 100%;
+  border-collapse: collapse;
+  margin: 1rem 0;
+}
+
+.markdown-content th,
+.markdown-content td {
+  padding: 0.75rem;
+  border: 1px solid var(--border);
+}
+
+.markdown-content th {
+  background: var(--bg);
+  font-weight: 600;
+}
+
+.footer {
+  border-top: 1px solid var(--border);
+  padding: 2rem 0;
+  margin-top: 4rem;
+  text-align: center;
+  color: var(--text-muted);
+}
+
+.footer a {
+  color: var(--primary);
+  text-decoration: none;
+}
+
+.footer a:hover {
+  text-decoration: underline;
+}
+

frontend/src/main.tsx

@@ -0,0 +1,11 @@
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import App from './App.tsx'
+import './index.css'
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>,
+)
+

frontend/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": ["src"],
+  "references": [{ "path": "./tsconfig.node.json" }]
+}
+

frontend/tsconfig.node.json

@@ -0,0 +1,11 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "skipLibCheck": true,
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["vite.config.ts"]
+}
+

frontend/vite.config.ts

@@ -0,0 +1,11 @@
+import { defineConfig } from 'vite'
+import react from '@vitejs/plugin-react'
+
+export default defineConfig({
+  plugins: [react()],
+  server: {
+    host: '0.0.0.0', // Allows access from any network interface
+    port: 5173,
+  },
+})
+

generated-docs/.gitkeep

@@ -0,0 +1,6 @@
+# Generated Documentation
+
+This folder contains documentation files generated by the Docs Extractor agent.
+
+Files are automatically saved here when you use the extraction tool.
+

main.ts

@@ -0,0 +1,184 @@
+import "@std/dotenv/load";
+import {
+  AnthropicModelProvider,
+  ZypherAgent,
+  createZypherContext,
+} from "@corespeed/zypher";
+import { eachValueFrom } from "rxjs-for-await";
+
+function getRequiredEnv(name: string): string {
+  const value = Deno.env.get(name);
+  if (!value) {
+    throw new Error(`Environment variable ${name} is not set`);
+  }
+  return value;
+}
+
+const prompt = `You are a documentation extraction agent. Extract and format documentation as clean markdown.
+
+RULES:
+- Output ONLY the final markdown document
+- Do NOT include meta-commentary like "I'll help you...", "Let me create...", "Perfect!", etc.
+- Do NOT narrate what you're doing
+- Start directly with the documentation title
+
+PROCESS:
+1. Crawl the main page (be fast - don't follow multiple links)
+2. Extract: installation, key concepts, 1-2 code examples
+3. Output the markdown document immediately
+
+FORMAT:
+# [Document Title]
+> Source: [original URL]
+
+[Your extracted content with headings, code blocks, and links]
+
+Be concise and practical. Output the document, not commentary about creating it.`;
+
+export class DocsExtractorAgent {
+  public zypher: ZypherAgent;
+
+  private constructor(agent: ZypherAgent) {
+    this.zypher = agent;
+  }
+
+  static async create() {
+    // Initialize the agent execution context
+    const workingDir = Deno.cwd();
+    const homeDir = Deno.env.get("HOME") || Deno.env.get("USERPROFILE") || workingDir;
+    
+    const zypherContext = await createZypherContext(workingDir, {
+      zypherDir: `${homeDir}/.zypher`,
+      enableBuiltInTools: true, // Explicitly enable built-in tools
+    });
+
+    // Create the agent with Anthropic model provider
+    const agent = new ZypherAgent(
+      zypherContext,
+      new AnthropicModelProvider({
+        apiKey: getRequiredEnv("ANTHROPIC_API_KEY"),
+      }),
+      {
+        customInstructions: prompt,
+      },
+    );
+
+    // Register Firecrawl MCP server
+    try {
+      await agent.mcp.registerServer({
+        id: "firecrawl",
+        type: "command",
+        command: {
+          command: "npx",
+          args: ["-y", "firecrawl-mcp"],
+          env: {
+            FIRECRAWL_API_KEY: getRequiredEnv("FIRECRAWL_API_KEY"),
+            FIRECRAWL_TIMEOUT: "30000", // 30 second timeout
+            FIRECRAWL_MAX_DEPTH: "1",   // Only crawl 1 level deep
+          },
+        },
+      });
+      
+      console.log("✅ Registered Firecrawl MCP server");
+    } catch (error) {
+      console.error("⚠️  Failed to register Firecrawl MCP server:", error);
+    }
+
+    return new DocsExtractorAgent(agent);
+  }
+
+  async extract(docsUrl: string, extractionType: string, model: string = "claude-sonnet-4-20250514") {
+    const userPrompt = `Extract from ${docsUrl}: ${extractionType}
+
+OUTPUT ONLY THE MARKDOWN DOCUMENT. No meta-commentary.
+
+Start with:
+# [Title for ${extractionType}]
+> Source: ${docsUrl}
+
+Then include:
+- Installation/setup (if relevant)
+- Key concepts
+- 1-2 code examples
+- Links to original docs
+
+Keep it concise and practical. Output the document directly.`;
+
+    console.log(`🔍 Starting extraction from ${docsUrl}...`);
+
+    const event$ = this.zypher.runTask(userPrompt, model);
+
+    let responseText = "";
+    const toolUses: Array<{ name: string; input: unknown }> = [];
+
+    for await (const event of eachValueFrom(event$)) {
+      switch (event.type) {
+        case "text":
+        case "message":
+          if (event.content?.text) {
+            responseText += event.content.text;
+          } else if (typeof event.content === "string") {
+            responseText += event.content;
+          }
+          break;
+
+        case "tool_use":
+        case "tool_use_input":
+          if (event.content?.name) {
+            toolUses.push({
+              name: event.content.name,
+              input: event.content.input,
+            });
+          } else if (event.tool || event.name) {
+            const toolName = event.tool || event.name;
+            const toolInput = event.input || event.content;
+            toolUses.push({ name: toolName, input: toolInput });
+          }
+          break;
+
+        case "tool_result":
+        case "tool_use_approved":
+        default:
+          break;
+      }
+    }
+
+    console.log("✅ Extraction completed!");
+    
+    // Save the response text to a file
+    try {
+      if (responseText) {
+        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+        const filename = `extraction-${timestamp}.md`;
+        const filepath = `./generated-docs/${filename}`;
+        await Deno.writeTextFile(filepath, responseText);
+        console.log(`📄 Saved: ${filename}`);
+      }
+    } catch (error) {
+      console.error("❌ Error saving file:", error);
+    }
+
+    return {
+      response: {
+        content: [
+          {
+            type: "text",
+            text: responseText,
+          },
+        ],
+      },
+      toolUses,
+    };
+  }
+
+  async cleanup() {
+  }
+}
+
+// Terminal mode: Run agent in interactive CLI
+if (import.meta.main) {
+  const { runAgentInTerminal } = await import("@corespeed/zypher");
+  const agent = await DocsExtractorAgent.create();
+  await runAgentInTerminal(agent.zypher, "claude-sonnet-4-20250514");
+  await agent.cleanup();
+}

server.ts

@@ -0,0 +1,106 @@
+import "@std/dotenv/load";
+import { Application, Router } from "@oak/oak";
+import { DocsExtractorAgent } from "./main.ts";
+
+const jobs = new Map<string, { status: string; result?: unknown; error?: string }>();
+
+const router = new Router();
+
+router
+  .post("/api/extract", async (ctx) => {
+    try {
+      const body = await ctx.request.body.json();
+      const { docsUrl, extractionType } = body;
+
+      if (!docsUrl || !extractionType) {
+        ctx.response.status = 400;
+        ctx.response.body = { error: "Missing required fields: docsUrl, extractionType" };
+        return;
+      }
+
+      const jobId = crypto.randomUUID();
+      jobs.set(jobId, { status: "processing" });
+
+      // Process extraction in background
+      (async () => {
+        try {
+          const agent = await DocsExtractorAgent.create();
+          const result = await agent.extract(docsUrl, extractionType, "claude-sonnet-4-20250514");
+          jobs.set(jobId, { status: "completed", result });
+          await agent.cleanup();
+        } catch (error) {
+          console.error("Extraction error:", error);
+          jobs.set(jobId, { 
+            status: "failed", 
+            error: error instanceof Error ? error.message : String(error)
+          });
+        }
+      })();
+
+      ctx.response.body = { jobId, status: "processing" };
+    } catch (error) {
+      console.error("API error:", error);
+      ctx.response.status = 500;
+      ctx.response.body = { 
+        error: error instanceof Error ? error.message : "Internal server error" 
+      };
+    }
+  })
+  .get("/api/job/:jobId", (ctx) => {
+    const jobId = ctx.params.jobId;
+    const job = jobs.get(jobId);
+
+    if (!job) {
+      ctx.response.status = 404;
+      ctx.response.body = { error: "Job not found" };
+      return;
+    }
+
+    ctx.response.body = job;
+  })
+  .get("/api/docs", async (ctx) => {
+    try {
+      const docs: Array<{ name: string; path: string }> = [];
+      for await (const entry of Deno.readDir("./generated-docs")) {
+        if (entry.isFile && entry.name.endsWith(".md")) {
+          docs.push({ name: entry.name, path: `./generated-docs/${entry.name}` });
+        }
+      }
+      ctx.response.body = { docs };
+    } catch (error) {
+      console.error("Error reading docs:", error);
+      ctx.response.body = { docs: [] };
+    }
+  })
+  .get("/api/docs/:filename", async (ctx) => {
+    try {
+      const filename = ctx.params.filename;
+      const content = await Deno.readTextFile(`./generated-docs/${filename}`);
+      ctx.response.headers.set("Content-Type", "text/markdown");
+      ctx.response.body = content;
+    } catch (error) {
+      ctx.response.status = 404;
+      ctx.response.body = { error: "Document not found" };
+    }
+  });
+
+const app = new Application();
+
+// CORS middleware
+app.use(async (ctx, next) => {
+  ctx.response.headers.set("Access-Control-Allow-Origin", "*");
+  ctx.response.headers.set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS");
+  ctx.response.headers.set("Access-Control-Allow-Headers", "Content-Type, Authorization");
+  if (ctx.request.method === "OPTIONS") {
+    ctx.response.status = 200;
+    return;
+  }
+  await next();
+});
+
+app.use(router.routes());
+app.use(router.allowedMethods());
+
+const PORT = 8000;
+console.log(`🚀 Docs Extractor API running on http://localhost:${PORT}`);
+await app.listen({ port: PORT });