PLUS: мультипроход со слиянием и проверкой нот по аудио
Browse files- plus/sse_proxy.py +109 -1
plus/sse_proxy.py
CHANGED
|
@@ -69,6 +69,52 @@ def apply_velocity(audio: tuple[str, bytes] | None, midi: bytes) -> tuple[bytes,
|
|
| 69 |
|
| 70 |
|
| 71 |
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
HOP_HEADERS = {"connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
|
| 73 |
"te", "trailers", "transfer-encoding", "upgrade", "host",
|
| 74 |
"content-length", "accept-encoding"}
|
|
@@ -123,6 +169,8 @@ class Proxy(BaseHTTPRequestHandler):
|
|
| 123 |
target_port = 8222
|
| 124 |
instruments: list[str] = []
|
| 125 |
measure_velocity = True
|
|
|
|
|
|
|
| 126 |
last_audio: tuple[str, bytes] | None = None
|
| 127 |
|
| 128 |
def log_message(self, fmt, *args):
|
|
@@ -171,8 +219,39 @@ class Proxy(BaseHTTPRequestHandler):
|
|
| 171 |
self.send_error(502, "upstream: %s" % problem)
|
| 172 |
return
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
self.send_response(answer.status)
|
| 175 |
-
stream =
|
| 176 |
for key, value in answer.getheaders():
|
| 177 |
low = key.lower()
|
| 178 |
if low in HOP_HEADERS or (stream and low == "content-length"):
|
|
@@ -197,6 +276,27 @@ class Proxy(BaseHTTPRequestHandler):
|
|
| 197 |
self.wfile.write(data)
|
| 198 |
upstream.close()
|
| 199 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
def stream_events(self, answer):
|
| 201 |
"""Переливаем SSE, вставляя ": ping" во время молчания модели."""
|
| 202 |
lines: queue.Queue = queue.Queue()
|
|
@@ -240,6 +340,12 @@ def main():
|
|
| 240 |
parser.add_argument("--listen", type=int, default=8223)
|
| 241 |
parser.add_argument("--target", type=int, default=8222)
|
| 242 |
parser.add_argument("--ping", type=float, default=PING_SECONDS)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
parser.add_argument("--no-velocity", action="store_true",
|
| 244 |
help="не мерить громкость, оставить константу 100")
|
| 245 |
parser.add_argument("--instruments", default=BAND,
|
|
@@ -248,6 +354,8 @@ def main():
|
|
| 248 |
Proxy.target_port = args.target
|
| 249 |
Proxy.ping_seconds = args.ping
|
| 250 |
Proxy.measure_velocity = not args.no_velocity
|
|
|
|
|
|
|
| 251 |
Proxy.instruments = [] if args.instruments.strip().lower() == "auto" else \
|
| 252 |
[n.strip() for n in args.instruments.split(",") if n.strip()]
|
| 253 |
server = ThreadingHTTPServer(("0.0.0.0", args.listen), Proxy)
|
|
|
|
| 69 |
|
| 70 |
|
| 71 |
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 72 |
+
import fusion # noqa: E402
|
| 73 |
+
import verify_notes # noqa: E402
|
| 74 |
+
def replace_instruments(body: bytes, content_type: str, names: list[str]) -> bytes:
|
| 75 |
+
"""Заменить поля instruments на нужный набор (для второго прохода)."""
|
| 76 |
+
marker = "boundary="
|
| 77 |
+
if "multipart/form-data" not in content_type or marker not in content_type:
|
| 78 |
+
return body
|
| 79 |
+
boundary = content_type.split(marker, 1)[1].split(";")[0].strip().strip('"')
|
| 80 |
+
chunks = body.split(b"--" + boundary.encode())
|
| 81 |
+
kept = []
|
| 82 |
+
for chunk in chunks:
|
| 83 |
+
if b'name="instruments"' in chunk:
|
| 84 |
+
continue
|
| 85 |
+
kept.append(chunk)
|
| 86 |
+
rebuilt = (b"--" + boundary.encode()).join(kept)
|
| 87 |
+
if names:
|
| 88 |
+
rebuilt = add_instruments(rebuilt, content_type, names)
|
| 89 |
+
return rebuilt
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def run_pass(host: str, port: int, path: str, body: bytes, content_type: str,
|
| 93 |
+
instruments: list[str]) -> str:
|
| 94 |
+
"""Один проход транскрипции к серверу, ответ как текст (SSE)."""
|
| 95 |
+
payload = replace_instruments(body, content_type, instruments)
|
| 96 |
+
connection = http.client.HTTPConnection(host, port, timeout=3600)
|
| 97 |
+
headers = {"Content-Type": content_type, "Content-Length": str(len(payload)),
|
| 98 |
+
"Accept-Encoding": "identity"}
|
| 99 |
+
connection.request("POST", path, body=payload, headers=headers)
|
| 100 |
+
answer = connection.getresponse()
|
| 101 |
+
text = answer.read().decode("utf-8", "ignore")
|
| 102 |
+
connection.close()
|
| 103 |
+
return text
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def build_midi(host: str, port: int, notes: list[dict]) -> bytes:
|
| 107 |
+
"""Собрать MIDI из слитых нот силами самого сервера (/export)."""
|
| 108 |
+
payload = json.dumps(fusion.notes_to_export(notes)).encode("utf-8")
|
| 109 |
+
connection = http.client.HTTPConnection(host, port, timeout=600)
|
| 110 |
+
connection.request("POST", "/export", body=payload,
|
| 111 |
+
headers={"Content-Type": "application/json"})
|
| 112 |
+
answer = connection.getresponse()
|
| 113 |
+
data = answer.read()
|
| 114 |
+
connection.close()
|
| 115 |
+
return data
|
| 116 |
+
|
| 117 |
+
|
| 118 |
HOP_HEADERS = {"connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
|
| 119 |
"te", "trailers", "transfer-encoding", "upgrade", "host",
|
| 120 |
"content-length", "accept-encoding"}
|
|
|
|
| 169 |
target_port = 8222
|
| 170 |
instruments: list[str] = []
|
| 171 |
measure_velocity = True
|
| 172 |
+
multi_pass: list[str] = []
|
| 173 |
+
energy_threshold = 6.0
|
| 174 |
last_audio: tuple[str, bytes] | None = None
|
| 175 |
|
| 176 |
def log_message(self, fmt, *args):
|
|
|
|
| 219 |
self.send_error(502, "upstream: %s" % problem)
|
| 220 |
return
|
| 221 |
|
| 222 |
+
content_type = (answer.getheader("Content-Type") or "")
|
| 223 |
+
if (self.multi_pass and method == "POST"
|
| 224 |
+
and self.path.split("?")[0] in ("/transcribe", "/transcribe/midi")
|
| 225 |
+
and content_type.startswith("text/event-stream")):
|
| 226 |
+
first = answer.read().decode("utf-8", "ignore")
|
| 227 |
+
upstream.close()
|
| 228 |
+
second = run_pass(self.target_host, self.target_port, self.path, body,
|
| 229 |
+
self.headers.get("Content-Type") or "", self.multi_pass)
|
| 230 |
+
notes, stats = fusion.merge([fusion.parse_sse(first), fusion.parse_sse(second)])
|
| 231 |
+
print("слияние: %s" % stats, flush=True)
|
| 232 |
+
notes = self.filter_by_audio(notes)
|
| 233 |
+
if self.path.split("?")[0] == "/transcribe/midi":
|
| 234 |
+
data = build_midi(self.target_host, self.target_port, notes)
|
| 235 |
+
if self.measure_velocity and data[:4] == b"MThd":
|
| 236 |
+
data, vstats = apply_velocity(self.last_audio, data)
|
| 237 |
+
print("velocity: %s" % vstats, flush=True)
|
| 238 |
+
self.send_response(200)
|
| 239 |
+
self.send_header("Content-Type", "audio/midi")
|
| 240 |
+
self.send_header("Content-Length", str(len(data)))
|
| 241 |
+
self.end_headers()
|
| 242 |
+
self.wfile.write(data)
|
| 243 |
+
return
|
| 244 |
+
payload = fusion.to_sse(notes)
|
| 245 |
+
self.send_response(200)
|
| 246 |
+
self.send_header("Content-Type", "text/event-stream")
|
| 247 |
+
self.send_header("Cache-Control", "no-cache")
|
| 248 |
+
self.send_header("Content-Length", str(len(payload)))
|
| 249 |
+
self.end_headers()
|
| 250 |
+
self.wfile.write(payload)
|
| 251 |
+
return
|
| 252 |
+
|
| 253 |
self.send_response(answer.status)
|
| 254 |
+
stream = content_type.startswith("text/event-stream")
|
| 255 |
for key, value in answer.getheaders():
|
| 256 |
low = key.lower()
|
| 257 |
if low in HOP_HEADERS or (stream and low == "content-length"):
|
|
|
|
| 276 |
self.wfile.write(data)
|
| 277 |
upstream.close()
|
| 278 |
|
| 279 |
+
def filter_by_audio(self, notes: list[dict]) -> list[dict]:
|
| 280 |
+
"""Оставить только то, что подтверждается энергией в записи."""
|
| 281 |
+
if not notes or not self.last_audio or self.energy_threshold <= 0:
|
| 282 |
+
return notes
|
| 283 |
+
import tempfile
|
| 284 |
+
name, blob = self.last_audio
|
| 285 |
+
suffix = ("." + name.rsplit(".", 1)[-1].lower()) if "." in name else ".wav"
|
| 286 |
+
try:
|
| 287 |
+
with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as handle:
|
| 288 |
+
handle.write(blob)
|
| 289 |
+
handle.flush()
|
| 290 |
+
kept, dropped, stats = verify_notes.verify(
|
| 291 |
+
notes, handle.name, threshold=self.energy_threshold)
|
| 292 |
+
except Exception as problem:
|
| 293 |
+
print("проверка по аудио не вышла: %s" % problem, flush=True)
|
| 294 |
+
return notes
|
| 295 |
+
print("проверка по аудио: %s" % stats, flush=True)
|
| 296 |
+
for note in kept:
|
| 297 |
+
note.pop("energy_ratio", None)
|
| 298 |
+
return kept
|
| 299 |
+
|
| 300 |
def stream_events(self, answer):
|
| 301 |
"""Переливаем SSE, вставляя ": ping" во время молчания модели."""
|
| 302 |
lines: queue.Queue = queue.Queue()
|
|
|
|
| 340 |
parser.add_argument("--listen", type=int, default=8223)
|
| 341 |
parser.add_argument("--target", type=int, default=8222)
|
| 342 |
parser.add_argument("--ping", type=float, default=PING_SECONDS)
|
| 343 |
+
parser.add_argument("--multi-pass", default="",
|
| 344 |
+
help="второй проход с этим набором инструментов и слияние,"
|
| 345 |
+
" например: chromatic_percussion,electric_bass,acoustic_bass")
|
| 346 |
+
parser.add_argument("--energy-threshold", type=float, default=6.0,
|
| 347 |
+
help="во сколько раз тон должен превышать соседние полутона;"
|
| 348 |
+
" 0 = не проверять")
|
| 349 |
parser.add_argument("--no-velocity", action="store_true",
|
| 350 |
help="не мерить громкость, оставить константу 100")
|
| 351 |
parser.add_argument("--instruments", default=BAND,
|
|
|
|
| 354 |
Proxy.target_port = args.target
|
| 355 |
Proxy.ping_seconds = args.ping
|
| 356 |
Proxy.measure_velocity = not args.no_velocity
|
| 357 |
+
Proxy.multi_pass = [n.strip() for n in args.multi_pass.split(",") if n.strip()]
|
| 358 |
+
Proxy.energy_threshold = args.energy_threshold
|
| 359 |
Proxy.instruments = [] if args.instruments.strip().lower() == "auto" else \
|
| 360 |
[n.strip() for n in args.instruments.split(",") if n.strip()]
|
| 361 |
server = ThreadingHTTPServer(("0.0.0.0", args.listen), Proxy)
|