Questionmarkboy commited on
Commit
3a6020f
·
verified ·
1 Parent(s): a4f8803

🧟 Frankenstein-3.0: Complete deployment

Browse files
Files changed (3) hide show
  1. README.md +54 -0
  2. app.py +316 -0
  3. requirements.txt +14 -0
README.md ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Frankenstein-3.0
3
+ emoji: 🧟
4
+ colorFrom: purple
5
+ colorTo: blue
6
+ sdk: gradio
7
+ sdk_version: 4.44.1
8
+ app_file: app.py
9
+ license: apache-2.0
10
+ ---
11
+
12
+ # 🧟 Frankenstein-3.0: The Unified AI Entity
13
+
14
+ A multi-modal AI orchestrator with an LLM-powered intelligent router.
15
+
16
+ ## Specialists
17
+ - **Chat** — Qwen3.5-4B (conversation + reasoning)
18
+ - **Code** — Qwen2.5-Coder-14B (programming)
19
+ - **Image** — SDXL (image generation)
20
+ - **Video** — MiniMax-H3 NF4 (video WITH audio!)
21
+
22
+ ## Tools
23
+ - **Web Search** — DuckDuckGo real-time search
24
+ - **Calculator** — SymPy safe math
25
+ - **Code Execution** — sandboxed Python runner
26
+ - **RAG** — upload .txt documents and query them
27
+ - **Memory** — persistent SQLite conversation history
28
+
29
+ ## Intelligent Router
30
+ LLM-powered intent detection with multi-step task chaining
31
+ (e.g. "draw a robot then animate it" → image → video).
32
+
33
+ ## Run Locally
34
+ ```bash
35
+ git clone https://huggingface.co/Questionmarkboy/frankenstein-3.0
36
+ cd frankenstein-3.0
37
+ pip install -r requirements.txt
38
+ python app.py
39
+ ```
40
+ Then open http://localhost:7860
41
+
42
+ ## Hardware Requirements
43
+ | Mode | Requirement |
44
+ |------|-------------|
45
+ | Chat + Code (CPU) | Any modern computer, 16GB RAM |
46
+ | Chat + Code (GPU) | 8GB+ VRAM |
47
+ | Image Generation | 12GB+ VRAM |
48
+ | Video Generation | 16GB+ VRAM recommended |
49
+
50
+ The app auto-detects CUDA. On CPU-only machines, chat/code work
51
+ and image/video return a friendly "GPU required" message.
52
+
53
+ ## License
54
+ Apache 2.0
app.py ADDED
@@ -0,0 +1,316 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import torch
3
+ import gc
4
+ import json
5
+ import re
6
+ import time
7
+ import os
8
+ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
9
+ import requests
10
+ from bs4 import BeautifulSoup
11
+ import sympy
12
+ import sqlite3
13
+
14
+ HAS_CUDA = torch.cuda.is_available()
15
+
16
+ # ============ PHASE 11: MEMORY SYSTEM (SQLite) ============
17
+ class MemorySystem:
18
+ def __init__(self, db_path="frankenstein_memory.db"):
19
+ self.conn = sqlite3.connect(db_path, check_same_thread=False)
20
+ c = self.conn.cursor()
21
+ c.execute("""
22
+ CREATE TABLE IF NOT EXISTS conversations (
23
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
24
+ user_input TEXT, specialist TEXT, response TEXT, timestamp REAL
25
+ )
26
+ """)
27
+ self.conn.commit()
28
+ def save_interaction(self, user_input, specialist, response):
29
+ c = self.conn.cursor()
30
+ c.execute("INSERT INTO conversations (user_input, specialist, response, timestamp) VALUES (?, ?, ?, ?)",
31
+ (user_input, specialist, response, time.time()))
32
+ self.conn.commit()
33
+ def get_recent_context(self, limit=3):
34
+ c = self.conn.cursor()
35
+ c.execute("SELECT user_input, specialist, response FROM conversations ORDER BY timestamp DESC LIMIT ?", (limit,))
36
+ return c.fetchall()[::-1]
37
+ def clear_memory(self):
38
+ c = self.conn.cursor()
39
+ c.execute("DELETE FROM conversations")
40
+ self.conn.commit()
41
+
42
+ # ============ PHASE 9: INTERNET ACCESS ============
43
+ class InternetSearch:
44
+ @staticmethod
45
+ def search_web(query, max_results=3):
46
+ try:
47
+ headers = {'User-Agent': 'Mozilla/5.0'}
48
+ r = requests.get(f"https://duckduckgo.com/html/?q={query}", headers=headers, timeout=10)
49
+ soup = BeautifulSoup(r.text, 'html.parser')
50
+ return [{"title": a.get_text(), "url": a.get('href')}
51
+ for a in soup.find_all('a', class_='result__a')[:max_results]]
52
+ except Exception as e:
53
+ return [{"error": str(e)}]
54
+
55
+ # ============ PHASE 10: TOOL SUITE ============
56
+ class ToolSuite:
57
+ @staticmethod
58
+ def calculate(expression):
59
+ try:
60
+ result = sympy.sympify(expression)
61
+ return float(result) if result.is_number else str(result)
62
+ except Exception as e:
63
+ return f"Error: {str(e)}"
64
+ @staticmethod
65
+ def execute_python(code, timeout=5):
66
+ import subprocess, tempfile
67
+ try:
68
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
69
+ f.write(code); temp_path = f.name
70
+ result = subprocess.run(['python', temp_path], capture_output=True, text=True, timeout=timeout)
71
+ os.unlink(temp_path)
72
+ return {"output": result.stdout or "", "error": result.stderr or "", "returncode": result.returncode}
73
+ except Exception as e:
74
+ return {"error": str(e)}
75
+
76
+ # ============ PHASE 8: RAG ============
77
+ class RAGSystem:
78
+ def __init__(self):
79
+ self.documents = []
80
+ def add_document(self, file):
81
+ try:
82
+ if file.name.endswith('.txt'):
83
+ with open(file.name, 'r') as f: content = f.read()
84
+ self.documents.append({"filename": os.path.basename(file.name), "content": content})
85
+ return f"Added {os.path.basename(file.name)} to knowledge base"
86
+ return "Only .txt files supported"
87
+ except Exception as e:
88
+ return f"Error: {str(e)}"
89
+ def search(self, query, max_results=2):
90
+ if not self.documents: return "No documents uploaded yet"
91
+ results = []; q = query.lower()
92
+ for doc in self.documents:
93
+ if q in doc["content"].lower():
94
+ idx = doc["content"].lower().find(q)
95
+ snippet = doc["content"][max(0, idx-100):min(len(doc["content"]), idx+300)]
96
+ results.append(f"**{doc['filename']}**: ...{snippet}...")
97
+ return "\n\n".join(results[:max_results]) if results else "No relevant information found"
98
+
99
+ # ============ MODEL MANAGER (Dynamic Load/Unload) ============
100
+ class ModelManager:
101
+ def __init__(self):
102
+ self.resident_model = None; self.resident_tokenizer = None
103
+ self.image_model = None; self.video_model = None
104
+ self.coder_model = None; self.coder_tokenizer = None
105
+ def load_resident(self):
106
+ if self.resident_model is None:
107
+ print("Loading resident model...", flush=True)
108
+ self.resident_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3.5-4B", trust_remote_code=True)
109
+ if HAS_CUDA:
110
+ quant = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4",
111
+ bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True)
112
+ self.resident_model = AutoModelForCausalLM.from_pretrained(
113
+ "Qwen/Qwen3.5-4B", quantization_config=quant, device_map="auto",
114
+ torch_dtype=torch.float16, trust_remote_code=True)
115
+ else:
116
+ self.resident_model = AutoModelForCausalLM.from_pretrained(
117
+ "Qwen/Qwen3.5-4B", torch_dtype=torch.bfloat16, device_map="cpu", trust_remote_code=True)
118
+ print("Resident loaded", flush=True)
119
+ return self.resident_model, self.resident_tokenizer
120
+ def unload_specialist(self, name):
121
+ if name == "image" and self.image_model is not None:
122
+ del self.image_model; self.image_model = None
123
+ elif name == "video" and self.video_model is not None:
124
+ del self.video_model; self.video_model = None
125
+ elif name == "coder" and self.coder_model is not None:
126
+ del self.coder_model; del self.coder_tokenizer
127
+ self.coder_model = None; self.coder_tokenizer = None
128
+ gc.collect()
129
+ if HAS_CUDA: torch.cuda.empty_cache()
130
+ def load_image_model(self):
131
+ if not HAS_CUDA: raise RuntimeError("GPU required for image generation")
132
+ if self.image_model is None:
133
+ from diffusers import StableDiffusionXLPipeline
134
+ self.image_model = StableDiffusionXLPipeline.from_pretrained(
135
+ "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16)
136
+ self.image_model.enable_model_cpu_offload()
137
+ return self.image_model
138
+ def load_video_model(self):
139
+ if not HAS_CUDA: raise RuntimeError("GPU required for video generation")
140
+ if self.video_model is None:
141
+ from diffsynth.pipelines.minimax_h3_audio_video import MiniMaxH3Pipeline, ModelConfig
142
+ vram_config = {
143
+ "offload_dtype": torch.bfloat16, "offload_device": "cpu",
144
+ "onload_dtype": torch.bfloat16, "onload_device": "cpu",
145
+ "preparing_dtype": torch.bfloat16, "preparing_device": "cuda",
146
+ "computation_dtype": torch.bfloat16, "computation_device": "cuda",
147
+ }
148
+ self.video_model = MiniMaxH3Pipeline.from_pretrained(
149
+ torch_dtype=torch.bfloat16, device="cuda",
150
+ model_configs=[
151
+ ModelConfig(model_id="DiffSynth-Studio/MiniMax-H3-NF4", origin_file_pattern="minimax-h3-fl2va-nf4.safetensors", **vram_config),
152
+ ModelConfig(model_id="DiffSynth-Studio/MiniMax-H3-NF4", origin_file_pattern="minimax-h3-text-encoder-nf4.safetensors", **vram_config),
153
+ ModelConfig(model_id="DiffSynth-Studio/MiniMax-H3-NF4", origin_file_pattern="video_vae_nf4.safetensors", **vram_config),
154
+ ModelConfig(model_id="DiffSynth-Studio/MiniMax-H3-NF4", origin_file_pattern="audio_vae_nf4.safetensors", **vram_config),
155
+ ],
156
+ processor_config=ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/processor/"))
157
+ return self.video_model
158
+ def load_coder(self):
159
+ if not HAS_CUDA: return None, None
160
+ if self.coder_model is None:
161
+ self.coder_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-14B-Instruct")
162
+ quant = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4",
163
+ bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True)
164
+ self.coder_model = AutoModelForCausalLM.from_pretrained(
165
+ "Qwen/Qwen2.5-Coder-14B-Instruct", quantization_config=quant,
166
+ device_map="auto", torch_dtype=torch.float16)
167
+ return self.coder_model, self.coder_tokenizer
168
+
169
+ # ============ INTELLIGENT ROUTER ============
170
+ class FrankensteinRouter:
171
+ def __init__(self, model_manager, memory_system):
172
+ self.mm = model_manager; self.memory = memory_system
173
+ self.resident, self.tokenizer = model_manager.load_resident()
174
+ self.device = next(self.resident.parameters()).device
175
+ self.internet = InternetSearch(); self.tools = ToolSuite(); self.rag = RAGSystem()
176
+ def _generate(self, prompt, max_tokens=150, temperature=0.1):
177
+ messages = [{"role": "user", "content": prompt}]
178
+ try:
179
+ text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)
180
+ except TypeError:
181
+ text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + " /no_think"
182
+ inputs = self.tokenizer(text, return_tensors="pt").to(self.device)
183
+ with torch.no_grad():
184
+ out = self.resident.generate(**inputs, max_new_tokens=max_tokens, temperature=temperature,
185
+ top_p=0.9, do_sample=True, pad_token_id=self.tokenizer.eos_token_id)
186
+ return self.tokenizer.decode(out[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True).strip()
187
+ def _parse_json(self, response):
188
+ try:
189
+ match = re.search(r'\{.*\}', response.replace('\n', ''), re.DOTALL)
190
+ return json.loads(match.group()) if match else json.loads(response)
191
+ except Exception:
192
+ return None
193
+ def route(self, user_input):
194
+ recent = self.memory.get_recent_context(3)
195
+ context = ""
196
+ if recent:
197
+ context = "Recent interactions:\n" + "\n".join([f"User: {r[0][:50]}... -> {r[1]}" for r in recent]) + "\n\n"
198
+ prompt = f"""You are the routing brain of Frankenstein-3.0. Route to the correct specialist.
199
+ {context}SPECIALISTS:
200
+ - "chat": Conversation, questions, explanations
201
+ - "code": Programming, debugging, scripts, games
202
+ - "image": Pictures, art, illustrations, logos, photos
203
+ - "video": Animations, clips, motion, films
204
+ - "search": Web search, current events, real-time info
205
+ - "calc": Mathematical calculations
206
+ - "execute": Run Python code
207
+ - "rag": Answer questions from uploaded documents
208
+
209
+ USER: "{user_input}"
210
+
211
+ Respond ONLY with JSON:
212
+ {{"specialist": "<name>", "confidence": <0.0-1.0>, "reasoning": "<brief>", "steps": [{{"specialist": "<name>", "prompt": "<refined prompt>"}}]}}"""
213
+ parsed = self._parse_json(self._generate(prompt, max_tokens=150))
214
+ if parsed:
215
+ return {"steps": parsed.get("steps", [{"specialist": parsed.get("specialist", "chat"), "prompt": user_input}]),
216
+ "confidence": parsed.get("confidence", 0.8), "reasoning": parsed.get("reasoning", "LLM routed")}
217
+ return {"steps": [{"specialist": "chat", "prompt": user_input}], "confidence": 0.5, "reasoning": "Fallback"}
218
+ def execute_step(self, spec, prompt):
219
+ result = ""; media_path = None
220
+ try:
221
+ if spec == "chat":
222
+ result = self._generate(prompt, max_tokens=300, temperature=0.7)
223
+ elif spec == "code":
224
+ coder, coder_tok = self.mm.load_coder()
225
+ if coder is not None:
226
+ device = next(coder.parameters()).device
227
+ messages = [{"role": "user", "content": f"Write clean Python code for: {prompt}"}]
228
+ text = coder_tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
229
+ inputs = coder_tok(text, return_tensors="pt").to(device)
230
+ with torch.no_grad():
231
+ out = coder.generate(**inputs, max_new_tokens=500, temperature=0.3, pad_token_id=coder_tok.eos_token_id)
232
+ code = coder_tok.decode(out[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True)
233
+ self.mm.unload_specialist("coder")
234
+ result = f"```python\n{code}\n```"
235
+ else:
236
+ result = "```python\n" + self._generate(f"Write clean Python code for: {prompt}", max_tokens=400, temperature=0.3) + "\n```"
237
+ elif spec == "image":
238
+ pipe = self.mm.load_image_model()
239
+ img = pipe(prompt, num_inference_steps=25, width=768, height=768).images[0]
240
+ media_path = f"/tmp/img_{int(time.time())}.png"; img.save(media_path)
241
+ self.mm.unload_specialist("image")
242
+ result = "Image generated successfully!"
243
+ elif spec == "video":
244
+ pipe = self.mm.load_video_model()
245
+ from diffsynth.utils.data.audio_video import write_video_audio
246
+ video, audio = pipe(prompt, height=256, width=448, num_frames=25, num_inference_steps=20,
247
+ use_gradient_checkpointing=True, use_gradient_checkpointing_offload=True)
248
+ media_path = f"/tmp/vid_{int(time.time())}.mp4"
249
+ write_video_audio(video, audio, media_path, fps=24, audio_sample_rate=32000)
250
+ self.mm.unload_specialist("video")
251
+ result = "Video generated successfully!"
252
+ elif spec == "search":
253
+ sr = self.internet.search_web(prompt)
254
+ result = "**Web Search Results:**\n" + "\n".join([f"- {r['title']}: {r['url']}" for r in sr if 'title' in r])
255
+ elif spec == "calc":
256
+ result = f"**Calculation:** {prompt} = {self.tools.calculate(prompt)}"
257
+ elif spec == "execute":
258
+ er = self.tools.execute_python(prompt)
259
+ result = f"**Code Execution:**\nOutput: {er.get('output', 'None')}\nError: {er.get('error', 'None')}"
260
+ elif spec == "rag":
261
+ result = f"**RAG Search:**\n{self.rag.search(prompt)}"
262
+ else:
263
+ result = self._generate(prompt, max_tokens=300, temperature=0.7)
264
+ except Exception as e:
265
+ result = f"Specialist '{spec}' error: {str(e)}"
266
+ self.memory.save_interaction(prompt, spec, result[:100])
267
+ return result, media_path
268
+
269
+ # ============ MAIN ============
270
+ print("Initializing Frankenstein-3.0...", flush=True)
271
+ mm = ModelManager(); memory = MemorySystem(); router = FrankensteinRouter(mm, memory)
272
+ print("System ready!", flush=True)
273
+
274
+ def frankenstein_chat(message, history):
275
+ decision = router.route(message)
276
+ route_path = " -> ".join([s["specialist"] for s in decision["steps"]])
277
+ output = f"**Router:** `{route_path.upper()}` (conf: {decision['confidence']:.2f})\n*{decision['reasoning']}*\n\n---\n\n"
278
+ media_files = []
279
+ for step in decision["steps"]:
280
+ result, media_path = router.execute_step(step["specialist"], step["prompt"])
281
+ output += result + "\n\n"
282
+ if media_path: media_files.append(media_path)
283
+ return output, (media_files if media_files else None)
284
+
285
+ def upload_document(file): return router.rag.add_document(file)
286
+ def clear_memory():
287
+ memory.clear_memory(); return "Memory cleared"
288
+
289
+ with gr.Blocks(theme=gr.themes.Soft(), title="Frankenstein-3.0") as demo:
290
+ gr.Markdown("# 🧟 Frankenstein-3.0: Unified AI Entity")
291
+ gr.Markdown("**Specialists:** Chat - Code - Image - Video - Web Search - Calculator - Code Execution - RAG")
292
+ with gr.Tabs():
293
+ with gr.Tab("Chat"):
294
+ chatbot = gr.Chatbot(type="messages", height=500)
295
+ msg = gr.Textbox(label="Command", placeholder="Try: Draw a cat, or Write a snake game")
296
+ with gr.Row():
297
+ send_btn = gr.Button("Execute", variant="primary"); clr_btn = gr.Button("Clear")
298
+ media_gallery = gr.Gallery(label="Generated Media", height=300)
299
+ def respond(message, history):
300
+ if not message.strip(): return "", history, None
301
+ history = history + [{"role": "user", "content": message}]
302
+ response, media = frankenstein_chat(message, history)
303
+ history = history + [{"role": "assistant", "content": response}]
304
+ return "", history, media
305
+ msg.submit(respond, [msg, chatbot], [msg, chatbot, media_gallery])
306
+ send_btn.click(respond, [msg, chatbot], [msg, chatbot, media_gallery])
307
+ clr_btn.click(lambda: [], None, chatbot)
308
+ with gr.Tab("Upload Documents (RAG)"):
309
+ file_upload = gr.File(label="Upload .txt file"); upload_btn = gr.Button("Add to Knowledge Base")
310
+ upload_status = gr.Textbox(label="Status")
311
+ upload_btn.click(upload_document, [file_upload], upload_status)
312
+ with gr.Tab("System"):
313
+ mem_btn = gr.Button("Clear Memory"); mem_status = gr.Textbox(label="Status")
314
+ mem_btn.click(clear_memory, outputs=mem_status)
315
+
316
+ demo.launch(share=False, server_name="0.0.0.0", server_port=7860)
requirements.txt ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ torch>=2.0.0
2
+ transformers>=4.40.0
3
+ bitsandbytes>=0.46.1
4
+ accelerate>=0.28.0
5
+ gradio>=4.44.0
6
+ diffusers>=0.25.0
7
+ safetensors>=0.4.0
8
+ pillow>=9.0.0
9
+ requests>=2.28.0
10
+ beautifulsoup4>=4.12.0
11
+ sympy>=1.12
12
+ sentencepiece>=0.1.99
13
+ protobuf>=3.20.0
14
+ git+https://github.com/modelscope/DiffSynth-Studio.git