k0880k commited on
Commit
6eb9193
·
1 Parent(s): a4ed6d1

feat: seed runtime genomes at ingestion

Browse files
Files changed (1) hide show
  1. core/runtime_genome.py +340 -0
core/runtime_genome.py ADDED
@@ -0,0 +1,340 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Runtime-genome inference for every SignalMesh ingestion path.
2
+
3
+ A runtime genome is *not* a container image and it is not a claim that SignalMesh
4
+ has reproduced an application. It is the portable seed from which that work can
5
+ start: what runtime families, manifests, services and capabilities are visible in
6
+ what just arrived, plus the evidence that caused each inference.
7
+
8
+ The registry attaches one to every accepted signal. Richer producers may supply
9
+ hints, and later runtime tracing can promote a seed into a verified closure.
10
+
11
+ Security rule: the mesh holds addresses and requirements, never the goods. This
12
+ module records environment-variable NAMES and opaque refs only; credential values,
13
+ cookies, bearer tokens and secret-looking metadata values are never copied into a
14
+ genome.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import re
19
+ from urllib.parse import urlparse
20
+ from typing import Any, Dict, Iterable, List, Optional, Set
21
+
22
+ SCHEMA = "signalmesh.runtime-genome/v1"
23
+
24
+ _SECRET_KEY = re.compile(
25
+ r"(?:^|[_-])(secret|password|passwd|token|cookie|authorization|api[_-]?key|"
26
+ r"private[_-]?key|client[_-]?secret|access[_-]?token|refresh[_-]?token)(?:$|[_-])",
27
+ re.I,
28
+ )
29
+
30
+ # Strong, cheap signatures. These describe conditions, not brands SignalMesh
31
+ # promises to recreate. More exact dependency closure belongs to runtime tracing.
32
+ _SIGNATURES = (
33
+ ("container", ("dockerfile", "docker-compose", "compose.yaml", "compose.yml", "oci image", "containerd", "podman")),
34
+ ("python", ("requirements.txt", "pyproject.toml", "setup.py", "pipfile", ".py ", "python ", "uv.lock")),
35
+ ("node", ("package.json", "package-lock.json", "pnpm-lock", "yarn.lock", "node.js", "npm ", "npx ", "tsconfig.json", ".tsx", ".ts ", ".mjs", ".cjs")),
36
+ ("jvm", ("pom.xml", "build.gradle", "gradlew", "java ", ".jar", ".war", "jvm")),
37
+ ("dotnet", (".csproj", ".sln", "dotnet ", "nuget", "clr")),
38
+ ("go", ("go.mod", "go.sum", "golang", " go ")),
39
+ ("rust", ("cargo.toml", "cargo.lock", "rustc", "cargo ")),
40
+ ("wasm", (".wasm", "wasi", "wasmtime", "webassembly")),
41
+ ("browser", ("playwright", "puppeteer", "selenium", "chromium", "chrome", "browser", "webview")),
42
+ ("accelerator", ("cuda", "cudnn", "rocm", "metal performance", "tpu")),
43
+ ("distributed", ("kubernetes", "k8s", "helm chart", "deployment.yaml", "statefulset", "service mesh")),
44
+ ("native", (".so ", ".dylib", ".dll", "ldd ", "otool ", "native binary", "executable")),
45
+ )
46
+
47
+ _LANGUAGE_BY_EXT = {
48
+ ".py": "python", ".pyw": "python",
49
+ ".js": "javascript", ".mjs": "javascript", ".cjs": "javascript",
50
+ ".ts": "typescript", ".tsx": "typescript",
51
+ ".java": "java", ".kt": "kotlin", ".scala": "scala",
52
+ ".cs": "csharp", ".fs": "fsharp",
53
+ ".go": "go", ".rs": "rust",
54
+ ".rb": "ruby", ".php": "php",
55
+ ".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp", ".cxx": "cpp",
56
+ ".swift": "swift", ".m": "objective-c", ".mm": "objective-cpp",
57
+ ".sh": "shell", ".bash": "shell", ".ps1": "powershell",
58
+ ".wasm": "webassembly",
59
+ }
60
+
61
+ _SERVICE_WORDS = {
62
+ "postgres": "postgres", "postgresql": "postgres", "mysql": "mysql",
63
+ "mariadb": "mariadb", "sqlite": "sqlite", "redis": "redis",
64
+ "mongodb": "mongodb", "rabbitmq": "rabbitmq", "kafka": "kafka",
65
+ "ollama": "ollama", "nginx": "nginx", "apache": "apache-httpd",
66
+ }
67
+
68
+ _REMOTE_SOURCE_TYPES = {
69
+ "harvest", "hydrated", "external", "feed", "rss", "rss_cp",
70
+ "feed_hydrator", "site_docs", "nicoli_catalog", "adys_pump",
71
+ }
72
+
73
+ _CAPABILITY_FIELDS = (
74
+ "capabilities", "actions", "operations", "commands", "tools", "skills",
75
+ "routes", "provides",
76
+ )
77
+
78
+ _SAFE_DECLARED_FIELDS = {
79
+ "runtime_families", "languages", "manifests", "services", "executables",
80
+ "capabilities", "env_names", "network_hosts", "entrypoints", "requires",
81
+ "materializers", "node_ref", "session_ref", "runtime_ref", "notes",
82
+ }
83
+
84
+
85
+ def _strings(value: Any, *, limit: int = 80) -> List[str]:
86
+ """Flatten names without turning arbitrary nested payloads into a second signal."""
87
+ out: List[str] = []
88
+ if isinstance(value, str):
89
+ v = value.strip()
90
+ if v:
91
+ out.append(v[:300])
92
+ elif isinstance(value, dict):
93
+ for k in list(value)[:limit]:
94
+ if isinstance(k, str) and k.strip():
95
+ out.append(k.strip()[:300])
96
+ elif isinstance(value, (list, tuple, set)):
97
+ for item in list(value)[:limit]:
98
+ if isinstance(item, str) and item.strip():
99
+ out.append(item.strip()[:300])
100
+ elif isinstance(item, dict):
101
+ name = item.get("name") or item.get("id") or item.get("capability")
102
+ if isinstance(name, str) and name.strip():
103
+ out.append(name.strip()[:300])
104
+ return out
105
+
106
+
107
+ def _safe_metadata_text(metadata: Dict[str, Any]) -> str:
108
+ parts: List[str] = []
109
+ for key, value in metadata.items():
110
+ if _SECRET_KEY.search(str(key)):
111
+ continue
112
+ if key in ("runtime_genome", "env", "environment"):
113
+ continue
114
+ if isinstance(value, (str, int, float, bool)):
115
+ parts.append(f"{key}={str(value)[:500]}")
116
+ elif isinstance(value, (list, tuple)) and len(value) <= 80:
117
+ parts.extend(str(x)[:200] for x in value if isinstance(x, (str, int, float)))
118
+ return " ".join(parts)
119
+
120
+
121
+ def _payload_text(data: Any) -> str:
122
+ if isinstance(data, str):
123
+ return data[:12000]
124
+ if isinstance(data, dict):
125
+ parts: List[str] = []
126
+ for key, value in list(data.items())[:120]:
127
+ if _SECRET_KEY.search(str(key)):
128
+ continue
129
+ parts.append(str(key))
130
+ if isinstance(value, (str, int, float, bool)):
131
+ parts.append(str(value)[:1000])
132
+ elif key in _CAPABILITY_FIELDS:
133
+ parts.extend(_strings(value))
134
+ return " ".join(parts)[:12000]
135
+ if isinstance(data, (list, tuple)):
136
+ return " ".join(_payload_text(x)[:800] for x in data[:20])[:12000]
137
+ return str(data)[:2000] if data is not None else ""
138
+
139
+
140
+ def _fileish(metadata: Dict[str, Any], files: Optional[Iterable[str]]) -> List[str]:
141
+ vals: List[str] = []
142
+ for key in ("file", "filename", "path", "manifest", "entrypoint"):
143
+ val = metadata.get(key)
144
+ if isinstance(val, str):
145
+ vals.append(val)
146
+ for key in ("files", "manifests"):
147
+ vals.extend(_strings(metadata.get(key)))
148
+ if files:
149
+ vals.extend(str(x) for x in files if x)
150
+ # Stable de-dupe while preserving producer order.
151
+ return list(dict.fromkeys(v[:500] for v in vals if v))[:200]
152
+
153
+
154
+ def _languages(files: Iterable[str], blob: str) -> List[str]:
155
+ found: Set[str] = set()
156
+ lower_blob = blob.lower()
157
+ for path in files:
158
+ low = path.lower()
159
+ for ext, lang in _LANGUAGE_BY_EXT.items():
160
+ if low.endswith(ext):
161
+ found.add(lang)
162
+ # Manifest evidence is stronger than prose mentioning a language name.
163
+ if "requirements.txt" in lower_blob or "pyproject.toml" in lower_blob:
164
+ found.add("python")
165
+ if "package.json" in lower_blob or "tsconfig.json" in lower_blob:
166
+ found.add("typescript" if "tsconfig.json" in lower_blob else "javascript")
167
+ if "go.mod" in lower_blob:
168
+ found.add("go")
169
+ if "cargo.toml" in lower_blob:
170
+ found.add("rust")
171
+ if "pom.xml" in lower_blob or "build.gradle" in lower_blob:
172
+ found.add("java")
173
+ return sorted(found)
174
+
175
+
176
+ def _manifests(files: Iterable[str], blob: str) -> List[str]:
177
+ known = (
178
+ "requirements.txt", "pyproject.toml", "setup.py", "pipfile", "uv.lock",
179
+ "package.json", "package-lock.json", "pnpm-lock.yaml", "yarn.lock", "tsconfig.json",
180
+ "pom.xml", "build.gradle", "gradle.properties", "go.mod", "go.sum",
181
+ "cargo.toml", "cargo.lock", "dockerfile", "docker-compose.yml", "docker-compose.yaml",
182
+ "compose.yml", "compose.yaml", ".csproj", ".sln",
183
+ )
184
+ lower_files = [x.lower() for x in files]
185
+ lower_blob = blob.lower()
186
+ found: Set[str] = set()
187
+ for name in known:
188
+ if any(f.endswith(name) for f in lower_files) or name in lower_blob:
189
+ found.add(name)
190
+ return sorted(found)
191
+
192
+
193
+ def _runtime_families(source_type: str, blob: str) -> List[str]:
194
+ low = f" {blob.lower()} "
195
+ found: Set[str] = set()
196
+ for family, signatures in _SIGNATURES:
197
+ if any(sig in low for sig in signatures):
198
+ found.add(family)
199
+ if (source_type or "").lower() in _REMOTE_SOURCE_TYPES:
200
+ found.add("remote")
201
+ return sorted(found)
202
+
203
+
204
+ def _services(blob: str) -> List[str]:
205
+ low = blob.lower()
206
+ return sorted({name for word, name in _SERVICE_WORDS.items() if word in low})
207
+
208
+
209
+ def _capabilities(data: Any, metadata: Dict[str, Any]) -> List[str]:
210
+ found: Set[str] = set()
211
+ for source in (metadata, data if isinstance(data, dict) else {}):
212
+ for key in _CAPABILITY_FIELDS:
213
+ for value in _strings(source.get(key)):
214
+ # Capability names are addresses, not essays.
215
+ if 1 <= len(value) <= 120 and not _SECRET_KEY.search(value):
216
+ found.add(value)
217
+ return sorted(found)[:200]
218
+
219
+
220
+ def _env_names(metadata: Dict[str, Any], data: Any) -> List[str]:
221
+ """Names are useful requirements; values are secret material and stay out."""
222
+ names: Set[str] = set()
223
+ for source in (metadata, data if isinstance(data, dict) else {}):
224
+ for key in ("env", "environment", "env_names"):
225
+ value = source.get(key)
226
+ if isinstance(value, dict):
227
+ names.update(str(k) for k in value if k)
228
+ elif isinstance(value, (list, tuple, set)):
229
+ names.update(str(x) for x in value if isinstance(x, str) and x)
230
+ return sorted(n[:160] for n in names if n)[:200]
231
+
232
+
233
+ def _network_hosts(metadata: Dict[str, Any]) -> List[str]:
234
+ hosts: Set[str] = set()
235
+ for key in ("source_url", "feed_url", "resolved_url", "origin", "url", "endpoint"):
236
+ value = metadata.get(key)
237
+ if not isinstance(value, str) or not value.startswith(("http://", "https://")):
238
+ continue
239
+ try:
240
+ host = urlparse(value).hostname
241
+ except Exception:
242
+ host = None
243
+ if host:
244
+ hosts.add(host.lower())
245
+ return sorted(hosts)
246
+
247
+
248
+ def _safe_declared(raw: Any) -> Dict[str, Any]:
249
+ if not isinstance(raw, dict):
250
+ return {}
251
+ out: Dict[str, Any] = {}
252
+ for key in _SAFE_DECLARED_FIELDS:
253
+ if key not in raw or _SECRET_KEY.search(key):
254
+ continue
255
+ value = raw[key]
256
+ if key == "env_names":
257
+ out[key] = _strings(value, limit=200)
258
+ elif key in {"runtime_families", "languages", "manifests", "services", "executables",
259
+ "capabilities", "network_hosts", "entrypoints", "materializers"}:
260
+ out[key] = _strings(value, limit=200)
261
+ elif key in {"node_ref", "session_ref", "runtime_ref", "notes"} and isinstance(value, str):
262
+ out[key] = value[:500]
263
+ elif key == "requires" and isinstance(value, dict):
264
+ safe_requires = {}
265
+ for req_key, req_value in list(value.items())[:100]:
266
+ if _SECRET_KEY.search(str(req_key)):
267
+ continue
268
+ if isinstance(req_value, (str, int, float, bool)):
269
+ safe_requires[str(req_key)[:100]] = str(req_value)[:300]
270
+ elif isinstance(req_value, (list, tuple)):
271
+ safe_requires[str(req_key)[:100]] = _strings(req_value, limit=100)
272
+ out[key] = safe_requires
273
+ return out
274
+
275
+
276
+ def infer(source_type: str, data: Any, metadata: Optional[Dict[str, Any]] = None,
277
+ files: Optional[Iterable[str]] = None) -> Dict[str, Any]:
278
+ """Return a deterministic runtime-genome seed for one accepted signal."""
279
+ meta = dict(metadata or {})
280
+ file_list = _fileish(meta, files)
281
+ payload = _payload_text(data)
282
+ meta_text = _safe_metadata_text(meta)
283
+ blob = " ".join(file_list + [payload, meta_text])
284
+
285
+ declared = _safe_declared(meta.get("runtime_genome"))
286
+ families = set(_runtime_families(source_type, blob))
287
+ families.update(declared.get("runtime_families", []))
288
+ languages = set(_languages(file_list, blob))
289
+ languages.update(declared.get("languages", []))
290
+ manifests = set(_manifests(file_list, blob))
291
+ manifests.update(declared.get("manifests", []))
292
+ services = set(_services(blob))
293
+ services.update(declared.get("services", []))
294
+ capabilities = set(_capabilities(data, meta))
295
+ capabilities.update(declared.get("capabilities", []))
296
+ env_names = set(_env_names(meta, data))
297
+ env_names.update(declared.get("env_names", []))
298
+ network_hosts = set(_network_hosts(meta))
299
+ network_hosts.update(declared.get("network_hosts", []))
300
+
301
+ evidence: List[str] = []
302
+ if file_list:
303
+ evidence.append("files:" + ",".join(file_list[:12]))
304
+ if manifests:
305
+ evidence.append("manifests:" + ",".join(sorted(manifests)[:12]))
306
+ if capabilities:
307
+ evidence.append("capability_fields")
308
+ if (source_type or "").lower() in _REMOTE_SOURCE_TYPES:
309
+ evidence.append("remote_source:" + (source_type or "unknown"))
310
+ if declared:
311
+ evidence.append("producer_runtime_hint")
312
+
313
+ genome: Dict[str, Any] = {
314
+ "schema": SCHEMA,
315
+ "closure_state": "seed",
316
+ "source_type": source_type or "unknown",
317
+ "runtime_families": sorted(families),
318
+ "languages": sorted(languages),
319
+ "manifests": sorted(manifests),
320
+ "services": sorted(services),
321
+ "capabilities": sorted(capabilities)[:200],
322
+ "env_names": sorted(env_names)[:200],
323
+ "network_hosts": sorted(network_hosts)[:100],
324
+ "evidence": evidence,
325
+ }
326
+ # Preserve safe producer detail without trusting it as verified closure.
327
+ for key in ("executables", "entrypoints", "requires", "materializers",
328
+ "node_ref", "session_ref", "runtime_ref", "notes"):
329
+ if key in declared:
330
+ genome[key] = declared[key]
331
+ return genome
332
+
333
+
334
+ def enrich_metadata(source_type: str, data: Any,
335
+ metadata: Optional[Dict[str, Any]] = None,
336
+ files: Optional[Iterable[str]] = None) -> Dict[str, Any]:
337
+ """Attach/refresh the genome while preserving every caller-owned metadata key."""
338
+ out = dict(metadata or {})
339
+ out["runtime_genome"] = infer(source_type, data, out, files)
340
+ return out