dl-proxy / app.py
jumannendange's picture
Rename app (3).py to app.py
d981d02 verified
Raw History Blame Contribute Delete
20 kB
import os
# Zima SSR ya Gradio (Node proxy) ili routes zetu za /mux/* zifike moja kwa moja.
os.environ.setdefault("GRADIO_SSR_MODE", "False")
import re
import shutil
import subprocess
import tempfile
import threading
import time
import uuid
import gradio as gr
import requests
from fastapi import Request
from fastapi.responses import FileResponse, JSONResponse
# ZeroGPU inahitaji angalau function moja yenye @spaces.GPU wakati wa startup.
# Hii ni placeholder tu; haiitwi kamwe, hivyo haitumii quota ya GPU.
try:
import spaces
@spaces.GPU
def _gpu_placeholder() -> str:
return "ok"
except ImportError:
pass
# HF inaweka hii automatic; tayari ina ".hf.space" ndani yake
SPACE_HOST = os.environ.get("SPACE_HOST", "")
SELF_URL = f"https://{SPACE_HOST}" if SPACE_HOST else os.environ.get("SELF_URL", "")
# Hiari: weka MUX_API_KEY kwenye Space Secrets, kisha edge function itume
# header "x-api-key" yenye thamani hiyo. Bila hiyo, mtu yeyote mwenye URL ya Space
# anaweza kutumia worker.
API_KEY = os.environ.get("MUX_API_KEY", "")
PING_INTERVAL_SECONDS = 300 # dakika 5
JOB_MAX_AGE_SECONDS = 2 * 3600 # futa kazi za zamani zaidi ya masaa 2
MAX_PARALLEL_JOBS = 2
WORK_ROOT = os.path.join(tempfile.gettempdir(), "jaynes_jobs")
os.makedirs(WORK_ROOT, exist_ok=True)
JOBS: dict = {}
JOBS_LOCK = threading.Lock()
SLOTS = threading.BoundedSemaphore(MAX_PARALLEL_JOBS)
UA = {"User-Agent": "Mozilla/5.0 (JaynesMuxWorker)"}
def safe_name(name: str) -> str:
name = (name or "video.mp4").strip()
name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', "_", name)[:150] or "video.mp4"
if not name.lower().endswith(".mp4"):
name += ".mp4"
return name
def download(url: str, dest: str) -> str:
with requests.get(url, stream=True, timeout=(15, 180), headers=UA) as r:
r.raise_for_status()
with open(dest, "wb") as f:
for chunk in r.iter_content(chunk_size=1024 * 1024):
if chunk:
f.write(chunk)
return dest
def set_job(job_id: str, **kw):
with JOBS_LOCK:
JOBS[job_id].update(kw)
# mov_text (subtitle track za mp4) zinahitaji msimbo wa LUGHA wa herufi 3
# hasa (ISO 639-2), la sivyo ffmpeg huandika field tupu ya lugha kimya kimya
# (bila error) — na hiyo husababisha baadhi ya player (VLC ya Android
# ikiwemo) kutoonyesha track hiyo kabisa hata ikiwa ipo kimuundo.
# Ramani hii inabadilisha misimbo ya kawaida ya herufi-2 (ISO 639-1) na
# misimbo isiyo ya kawaida kutoka kwa baadhi ya API, kuwa herufi-3 sahihi.
_LANG_FIXUPS = {
"aa": "aar", "ab": "abk", "af": "afr", "ak": "aka", "am": "amh",
"ar": "ara", "as": "asm", "ay": "aym", "az": "aze", "ba": "bak",
"be": "bel", "bg": "bul", "bi": "bis", "bn": "ben", "bo": "bod",
"br": "bre", "bs": "bos", "ca": "cat", "ce": "che", "co": "cos",
"cs": "ces", "cy": "cym", "da": "dan", "de": "deu", "dv": "div",
"dz": "dzo", "el": "ell", "en": "eng", "eo": "epo", "es": "spa",
"et": "est", "eu": "eus", "fa": "fas", "ff": "ful", "fi": "fin",
"fil": "fil", "fj": "fij", "fo": "fao", "fr": "fra", "fy": "fry",
"ga": "gle", "gd": "gla", "gl": "glg", "gn": "grn", "gu": "guj",
"ha": "hau", "he": "heb", "hi": "hin", "hr": "hrv", "ht": "hat",
"hu": "hun", "hy": "hye", "id": "ind", "ig": "ibo", "in": "ind",
"is": "isl", "it": "ita", "iu": "iku", "iw": "heb", "ja": "jpn",
"jv": "jav", "jw": "jav", "ka": "kat", "kk": "kaz", "km": "khm",
"kn": "kan", "ko": "kor", "ks": "kas", "ku": "kur", "ky": "kir",
"la": "lat", "lb": "ltz", "lg": "lug", "ln": "lin", "lo": "lao",
"lt": "lit", "lv": "lav", "mg": "mlg", "mi": "mri", "mk": "mkd",
"ml": "mal", "mn": "mon", "mr": "mar", "ms": "msa", "mt": "mlt",
"my": "mya", "ne": "nep", "nl": "nld", "no": "nor", "ny": "nya",
"om": "orm", "or": "ori", "pa": "pan", "pl": "pol", "ps": "pus",
"pt": "por", "qu": "que", "rn": "run", "ro": "ron", "ru": "rus",
"rw": "kin", "sa": "san", "sd": "snd", "si": "sin", "sk": "slk",
"sl": "slv", "sm": "smo", "sn": "sna", "so": "som", "sq": "sqi",
"sr": "srp", "st": "sot", "su": "sun", "sv": "swe", "sw": "swa",
"ta": "tam", "te": "tel", "tg": "tgk", "th": "tha", "ti": "tir",
"tk": "tuk", "tl": "fil", "to": "ton", "tr": "tur", "ts": "tso",
"tt": "tat", "ug": "uig", "uk": "ukr", "ur": "urd", "uz": "uzb",
"vi": "vie", "wo": "wol", "xh": "xho", "yi": "yid", "yo": "yor",
"za": "zha", "zh": "zho", "zu": "zul",
}
def _norm_lang(code: str) -> str:
code = (code or "und").strip().lower()
code = re.split(r"[_-]", code)[0] or "und"
code = re.sub(r"[^a-z]", "", code)
if len(code) == 2:
code = _LANG_FIXUPS.get(code, "und")
elif len(code) != 3:
code = "und"
return code or "und"
def _sub_ext(path: str) -> str:
# Tambua WebVTT au SRT kwa maudhui, si kwa jina la faili
with open(path, "rb") as f:
head = f.read(16).decode("utf-8", "ignore").lstrip("\ufeff").lstrip()
return ".vtt" if head.upper().startswith("WEBVTT") else ".srt"
# ---- Usafishaji wa manukuu kabla ya ffmpeg ----
# CHANZO CHA TATIZO (English tupu kwenye VLC): baadhi ya faili za manukuu ni
# mchanganyiko mbovu — kichwa cha "WEBVTT", herufi ya BOM katikati ya faili, na
# cue za mwanzo zikiwa kwa mtindo wa SRT ("1" + muda wenye koma 00:00:19,079)
# huku nyingine zikitumia nukta. _sub_ext() ilikiona "WEBVTT" na kukipa jina
# .vtt, hivyo ffmpeg ilitumia WebVTT demuxer kali, ikashindwa kusoma cue
# zote, na track ya mov_text ikatengenezwa BILA maandishi. Suluhisho: geuza
# kila manukuu kuwa SRT safi yenye namba kabla ya kumpa ffmpeg.
_TS = re.compile(r"(\d{1,2}):(\d\d):(\d\d)[,.](\d{1,3})\s*-->\s*(\d{1,2}):(\d\d):(\d\d)[,.](\d{1,3})")
def _decode_sub(raw: bytes) -> str:
# UTF-16 (BOM) -> UTF-8 -> Windows-1252, kwa mpangilio huo
if raw[:2] in (b"\xff\xfe", b"\xfe\xff"):
return raw.decode("utf-16", errors="replace")
try:
return raw.decode("utf-8")
except UnicodeDecodeError:
return raw.decode("cp1252", errors="replace")
def _fmt_ts(h, m, s, ms):
return f"{int(h):02d}:{m}:{s},{ms.ljust(3, '0')[:3]}"
def clean_srt(raw: bytes) -> str:
"""Geuza manukuu yoyote (SRT, WEBVTT, mchanganyiko wenye BOM/koma/nukta)
kuwa SRT safi yenye namba. Inarudisha '' kama hakuna cue hata moja
(mfano faili la ASS) ili mwito aendelee na njia ya zamani."""
text = _decode_sub(raw).replace("\ufeff", "").replace("\r\n", "\n").replace("\r", "\n")
lines = text.split("\n")
cues, i = [], 0
while i < len(lines):
m = _TS.search(lines[i])
if not m:
i += 1
continue
i += 1
body = []
while i < len(lines) and lines[i].strip():
if _TS.search(lines[i]):
break
if lines[i].strip().isdigit() and i + 1 < len(lines) and _TS.search(lines[i + 1]):
break
body.append(lines[i])
i += 1
txt = re.sub(r"\{\\[^}]*\}", "", "\n".join(body))
txt = re.sub(r"</?font[^>]*>", "", txt, flags=re.I).strip()
if txt:
g = m.groups()
cues.append(f"{_fmt_ts(*g[0:4])} --> {_fmt_ts(*g[4:8])}\n{txt}")
if not cues:
return ""
return "\n\n".join(f"{n}\n{c}" for n, c in enumerate(cues, 1)) + "\n"
def probe_output(path: str) -> dict:
# Inatoa ffprobe -show_streams/-show_format kama JSON, ili tuweze kuthibitisha
# language/codec_tag/disposition ya subtitle track bila kupakua faili kwenye simu.
import json
r = subprocess.run(
["ffprobe", "-v", "error", "-print_format", "json",
"-show_streams", "-show_format", path],
capture_output=True, text=True, timeout=30,
)
try:
return json.loads(r.stdout)
except Exception:
return {"error": (r.stderr or "ffprobe imeshindwa")[-500:]}
def run_job(job_id: str, video_url: str, subtitles: list):
# subtitles: [{"url": "...", "language": "sw", "label": "Kiswahili"}, ...]
job_dir = os.path.join(WORK_ROOT, job_id)
video_path = os.path.join(job_dir, "input.mp4")
output_path = os.path.join(job_dir, "output.mp4")
sub_paths = []
try:
with SLOTS:
os.makedirs(job_dir, exist_ok=True)
set_job(job_id, status="downloading")
download(video_url, video_path)
for i, s in enumerate(subtitles):
raw = os.path.join(job_dir, f"subs{i}.raw")
download(s["url"], raw)
# Safisha kuwa SRT safi; kama haisomeki (mfano ASS), tumia njia ya zamani.
with open(raw, "rb") as f:
cleaned = clean_srt(f.read())
if cleaned:
sub_path = os.path.join(job_dir, f"subs{i}.srt")
with open(sub_path, "w", encoding="utf-8") as f:
f.write(cleaned)
os.remove(raw)
else:
ext = _sub_ext(raw)
sub_path = os.path.join(job_dir, f"subs{i}{ext}")
os.replace(raw, sub_path)
sub_paths.append({
"path": sub_path,
"language": _norm_lang(s.get("language")),
"label": s.get("label") or s.get("language") or f"Sub {i + 1}",
})
set_job(job_id, status="muxing")
# Soft subtitles: video na audio zinakopiwa bila kubadilishwa,
# kila manukuu yanawekwa kama track yake ya mov_text (yote
# yanaweza kuwashwa/kuzimwa/kuchaguliwa kwenye player).
cmd = ["ffmpeg", "-y", "-i", video_path]
for s in sub_paths:
cmd += ["-i", s["path"]]
cmd += ["-map", "0:v", "-map", "0:a?"]
for i in range(len(sub_paths)):
cmd += ["-map", f"{i + 1}:0"]
# Zuia data/chapter track yoyote ya chanzo (mfano "text"/tx3g track
# ya HandBrake ya zamani inayounganishwa na video kupitia "tref")
# kuvutwa kiotomatiki na -map 0:v -c:v copy hata bila kutajwa wazi.
cmd += ["-map", "-0:d"]
# MUHIMU: track ya "chapter" (QuickTime 'text'/tx3g inayounganishwa
# kupitia tref) HAIDHIBITIWI na -map kabisa — ffmpeg kwa default
# inanakili chapters za input ya kwanza (-map_chapters 0 default),
# na mov muxer inaandika hizo kama stream ya ziada. Zima kabisa.
cmd += ["-map_chapters", "-1"]
cmd += ["-c:v", "copy", "-c:a", "copy", "-c:s", "mov_text"]
for i, s in enumerate(sub_paths):
cmd += [f"-metadata:s:s:{i}", f"language={s['language']}"]
cmd += [f"-metadata:s:s:{i}", f"title={s['label']}"]
if i == 0:
cmd += [f"-disposition:s:{i}", "default"]
# JARIBIO LA SUBTITLE: -movflags +faststart imeondolewa kwa muda —
# tunachunguza kama ku-rewrite moov atom (faststart) ndiko
# kunakopoteza/kuharibu track ya subtitle kwenye VLC (Android),
# ingawa ffprobe bado inaiona. Ukithibitisha kuwa ndiyo tatizo,
# rejesha faststart kama HATUA TOFAUTI (remux ya pili, "-c copy"
# peke yake) baada ya mux hii, badala ya kuichanganya hapa.
cmd += [output_path]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=900)
if result.returncode != 0:
raise RuntimeError("ffmpeg imeshindwa: " + result.stderr[-300:])
for p in [video_path] + [s["path"] for s in sub_paths]:
try:
os.remove(p)
except OSError:
pass
set_job(job_id, status="done", path=output_path, probe=probe_output(output_path))
except Exception as e:
shutil.rmtree(job_dir, ignore_errors=True)
set_job(job_id, status="error", error=str(e)[:400])
# ---- Routes (zinaongezwa kwenye FastAPI app ya Gradio) ----
def _authorized(request: Request) -> bool:
return not API_KEY or request.headers.get("x-api-key") == API_KEY
async def mux_start(request: Request):
if not _authorized(request):
return JSONResponse({"error": "Hairuhusiwi."}, status_code=401)
try:
body = await request.json()
except Exception:
return JSONResponse({"error": "JSON si sahihi."}, status_code=400)
video_url = (body.get("videoUrl") or "").strip()
# Inakubali "subtitleUrls": [{"url","language","label"}, ...] (lugha zote),
# au bado "subtitleUrl" moja (backward-compatible na wateja wa zamani).
raw_subs = body.get("subtitleUrls")
if not isinstance(raw_subs, list):
raw_subs = []
subtitles = []
for s in raw_subs:
if isinstance(s, str):
u = s.strip()
if u:
subtitles.append({"url": u, "language": "und", "label": None})
elif isinstance(s, dict):
u = (s.get("url") or "").strip()
if u:
subtitles.append({"url": u, "language": s.get("language"), "label": s.get("label")})
if not subtitles:
legacy = (body.get("subtitleUrl") or "").strip()
if legacy:
subtitles = [{"url": legacy, "language": "und", "label": None}]
if not video_url.startswith(("http://", "https://")):
return JSONResponse({"error": "Weka videoUrl (http/https)."}, status_code=400)
for s in subtitles:
if not s["url"].startswith(("http://", "https://")):
return JSONResponse({"error": "subtitleUrls zote lazima ziwe http/https."}, status_code=400)
if len(subtitles) > 20:
return JSONResponse({"error": "Idadi kubwa mno ya subtitles (upeo 20)."}, status_code=400)
job_id = uuid.uuid4().hex
with JOBS_LOCK:
JOBS[job_id] = {
"status": "queued", "error": None, "path": None,
"name": safe_name(body.get("filename")), "created": time.time(),
}
threading.Thread(target=run_job, args=(job_id, video_url, subtitles), daemon=True).start()
return {"jobId": job_id}
async def mux_status(job_id: str, request: Request):
if not _authorized(request):
return JSONResponse({"error": "Hairuhusiwi."}, status_code=401)
with JOBS_LOCK:
job = dict(JOBS.get(job_id) or {})
if not job:
return JSONResponse(
{"status": "error", "error": "Kazi haipatikani (Space imeanza upya au imeisha muda). Jaribu tena."},
status_code=404,
)
out = {"status": job["status"]}
if job["status"] == "done":
out["resultPath"] = f"/mux/file/{job_id}"
if job["status"] == "error":
out["error"] = job["error"]
return out
# Hakuna API key hapa: job id ni ndefu na haiwezi kubashiriwa, na browser
# ya mtumiaji inafungua link hii moja kwa moja.
async def mux_file(job_id: str):
with JOBS_LOCK:
job = dict(JOBS.get(job_id) or {})
if job.get("status") != "done" or not job.get("path") or not os.path.exists(job["path"]):
return JSONResponse({"error": "Faili halipatikani."}, status_code=404)
# filename= inaweka Content-Disposition: attachment, hivyo simu inadownload
# badala ya kucheza video kwenye tab.
return FileResponse(job["path"], media_type="video/mp4", filename=job["name"])
async def mux_probe_url(request: Request):
# Debug ya jumla: pakua URL yoyote (chanzo cha video au subtitle) kwenye
# /tmp na urudishe ffprobe yake, bila kuchanganya na chochote. Inasaidia
# kubaini kama stream isiyotarajiwa (mfano track ya "text"/data) tayari
# ipo kwenye faili chanzo kabla hata ya kuchanganya na ffmpeg.
if not _authorized(request):
return JSONResponse({"error": "Hairuhusiwi."}, status_code=401)
url = request.query_params.get("url", "").strip()
if not url.startswith(("http://", "https://")):
return JSONResponse({"error": "Weka ?url=http(s)://..."}, status_code=400)
tmp_path = os.path.join(tempfile.gettempdir(), f"probe_{uuid.uuid4().hex}")
try:
download(url, tmp_path)
return probe_output(tmp_path)
except Exception as e:
return JSONResponse({"error": str(e)[:400]}, status_code=502)
finally:
try:
os.remove(tmp_path)
except OSError:
pass
async def mux_probe(job_id: str, request: Request):
# ffprobe kamili ya faili la mwisho — angalia hasa tags.language na
# codec_tag_string ("tx3g") kwa kila stream ya subtitle, bila kupakua faili.
if not _authorized(request):
return JSONResponse({"error": "Hairuhusiwi."}, status_code=401)
with JOBS_LOCK:
job = dict(JOBS.get(job_id) or {})
if not job:
return JSONResponse({"error": "Kazi haipatikani."}, status_code=404)
return job.get("probe", {"error": "Probe haipo bado (job bado haijakamilika au ilishindwa)."})
async def ffmpeg_version(request: Request):
# Thibitisha ffmpeg gani hasa inaendesha ndani ya Space hii sasa hivi.
if not _authorized(request):
return JSONResponse({"error": "Hairuhusiwi."}, status_code=401)
r = subprocess.run(["ffmpeg", "-version"], capture_output=True, text=True)
first_line = (r.stdout or r.stderr or "").splitlines()[:1]
return {"version": first_line[0] if first_line else "haijulikani"}
async def debug_info(request: Request):
# Thibitisha kwamba code inayoendesha kwenye Space ndiyo iliyopandishwa hivi
# karibuni (husaidia kubaini deployment ya zamani iliyokwama kwenye cache).
if not _authorized(request):
return JSONResponse({"error": "Hairuhusiwi."}, status_code=401)
import hashlib
with open(__file__, "rb") as f:
content = f.read()
return {
"app_sha1": hashlib.sha1(content).hexdigest()[:12],
"app_mtime": os.path.getmtime(__file__),
}
def cleanup_old_jobs():
now = time.time()
with JOBS_LOCK:
old = [j for j, v in JOBS.items() if now - v["created"] > JOB_MAX_AGE_SECONDS]
for j in old:
JOBS.pop(j, None)
try:
for name in os.listdir(WORK_ROOT):
path = os.path.join(WORK_ROOT, name)
try:
if now - os.path.getmtime(path) > JOB_MAX_AGE_SECONDS:
shutil.rmtree(path, ignore_errors=True)
except OSError:
pass
except OSError:
pass
def health_check() -> str:
return "ok"
# ---- Kuzuia Space isilale + usafi ----
def keep_alive_loop():
while True:
time.sleep(PING_INTERVAL_SECONDS)
cleanup_old_jobs()
if not SELF_URL:
continue
try:
requests.get(SELF_URL, timeout=15)
except Exception:
pass
threading.Thread(target=keep_alive_loop, daemon=True).start()
with gr.Blocks(title="JAYNES MAX — Soft Subtitle Mux Worker") as demo:
gr.Markdown("## JAYNES MAX — Soft Subtitle Mux Worker")
gr.Markdown(
"Worker hii haina fomu. Inatumiwa na edge function kupitia "
"`POST /mux/start`, `GET /mux/status/{jobId}` na `GET /mux/file/{jobId}`."
)
health_out = gr.Textbox(visible=False)
demo.load(fn=health_check, outputs=health_out, api_name="health")
demo.queue(max_size=20)
demo.launch(prevent_thread_lock=True)
# Ongeza routes zetu mbele ya routes za Gradio ili zipate kipaumbele
_fastapi = demo.app
_before = len(_fastapi.router.routes)
_fastapi.add_api_route("/mux/start", mux_start, methods=["POST"])
_fastapi.add_api_route("/mux/status/{job_id}", mux_status, methods=["GET"])
_fastapi.add_api_route("/mux/file/{job_id}", mux_file, methods=["GET"])
_fastapi.add_api_route("/mux/probe/{job_id}", mux_probe, methods=["GET"])
_fastapi.add_api_route("/mux/probe-url", mux_probe_url, methods=["GET"])
_fastapi.add_api_route("/mux/ffmpeg-version", ffmpeg_version, methods=["GET"])
_fastapi.add_api_route("/mux/debug-info", debug_info, methods=["GET"])
_new = _fastapi.router.routes[_before:]
_fastapi.router.routes[:] = _new + _fastapi.router.routes[:_before]
demo.block_thread()