Spaces:
Running
Running
Copy-first architecture: light fallback not fixed720; hysteresis 45s; relay_state in UI; no MediaMTX
Browse files- Dockerfile +5 -2
- app.py +8 -0
- index.html +2 -1
- relay.py +20 -2
- youtube_manager.py +49 -2
Dockerfile
CHANGED
|
@@ -18,19 +18,22 @@ ENV TZ=Asia/Baghdad \
|
|
| 18 |
YOUTUBE_TITLE_TEMPLATE="{broadcaster} | {date} {time}" \
|
| 19 |
YOUTUBE_SEND_MODE=auto \
|
| 20 |
YOUTUBE_AUTO_TRANSCODE=1 \
|
|
|
|
|
|
|
|
|
|
| 21 |
YOUTUBE_AUDIO_PROFILE=aac_low \
|
| 22 |
YOUTUBE_AUDIO_BITRATE=128k \
|
| 23 |
YOUTUBE_AUDIO_CHANNELS=2 \
|
| 24 |
YOUTUBE_GOP=60 \
|
| 25 |
YOUTUBE_INPUT_RECOVERY=1 \
|
| 26 |
YOUTUBE_PREFER_H264=1 \
|
| 27 |
-
YOUTUBE_CODEC_FALLBACK_MODE=
|
| 28 |
YOUTUBE_RETRY_MIN=1 \
|
| 29 |
YOUTUBE_RETRY_MAX=60 \
|
| 30 |
YOUTUBE_SOURCE_LIMIT=6 \
|
| 31 |
YOUTUBE_FORMAT_GUARD=on \
|
| 32 |
YOUTUBE_FLAP_LIMIT=4 \
|
| 33 |
-
YOUTUBE_FLAP_FALLBACK_MODE=
|
| 34 |
YOUTUBE_X264_PRESET=veryfast \
|
| 35 |
YOUTUBE_CRF=22 \
|
| 36 |
YOUTUBE_FIFO_TIMESHIFT=0 \
|
|
|
|
| 18 |
YOUTUBE_TITLE_TEMPLATE="{broadcaster} | {date} {time}" \
|
| 19 |
YOUTUBE_SEND_MODE=auto \
|
| 20 |
YOUTUBE_AUTO_TRANSCODE=1 \
|
| 21 |
+
YOUTUBE_LIGHT_MAX=1 \
|
| 22 |
+
YOUTUBE_LIGHT_BITRATE=1800k \
|
| 23 |
+
YOUTUBE_COPY_RECOVERY_SECONDS=45 \
|
| 24 |
YOUTUBE_AUDIO_PROFILE=aac_low \
|
| 25 |
YOUTUBE_AUDIO_BITRATE=128k \
|
| 26 |
YOUTUBE_AUDIO_CHANNELS=2 \
|
| 27 |
YOUTUBE_GOP=60 \
|
| 28 |
YOUTUBE_INPUT_RECOVERY=1 \
|
| 29 |
YOUTUBE_PREFER_H264=1 \
|
| 30 |
+
YOUTUBE_CODEC_FALLBACK_MODE=light \
|
| 31 |
YOUTUBE_RETRY_MIN=1 \
|
| 32 |
YOUTUBE_RETRY_MAX=60 \
|
| 33 |
YOUTUBE_SOURCE_LIMIT=6 \
|
| 34 |
YOUTUBE_FORMAT_GUARD=on \
|
| 35 |
YOUTUBE_FLAP_LIMIT=4 \
|
| 36 |
+
YOUTUBE_FLAP_FALLBACK_MODE=light \
|
| 37 |
YOUTUBE_X264_PRESET=veryfast \
|
| 38 |
YOUTUBE_CRF=22 \
|
| 39 |
YOUTUBE_FIFO_TIMESHIFT=0 \
|
app.py
CHANGED
|
@@ -596,6 +596,8 @@ def _relay_snapshot(user):
|
|
| 596 |
'relay_dest':slot.get('dest') if slot else '',
|
| 597 |
'restarts':hub.restart_count(user),
|
| 598 |
'send_mode':snap.get('send_mode') or '',
|
|
|
|
|
|
|
| 599 |
}
|
| 600 |
|
| 601 |
def _cgroup_usage_ns():
|
|
@@ -820,6 +822,8 @@ def health_public():
|
|
| 820 |
'last_error':tsk.get('last_error') or '','source_age':rs.get('source_age'),
|
| 821 |
'output_seconds':rs.get('output_seconds') or 0,
|
| 822 |
'send_mode':rs.get('send_mode') or '',
|
|
|
|
|
|
|
| 823 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 824 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 825 |
'youtube_status':str(ye.get('youtube_status') or '')})
|
|
@@ -934,6 +938,10 @@ def state():
|
|
| 934 |
'last_data_time':rs.get('last_data_time'),'relay_alive':rs.get('relay_alive'),
|
| 935 |
'session_title':rs.get('session_title') or '','restarts':rs.get('restarts') or 0,
|
| 936 |
'source_age':rs.get('source_age'),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 937 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 938 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 939 |
'youtube_status':str(ye.get('youtube_status') or ('relay' if rs.get('relay_alive') else (t.get('state') or '')))})
|
|
|
|
| 596 |
'relay_dest':slot.get('dest') if slot else '',
|
| 597 |
'restarts':hub.restart_count(user),
|
| 598 |
'send_mode':snap.get('send_mode') or '',
|
| 599 |
+
'relay_state':snap.get('relay_state') or '',
|
| 600 |
+
'transcode_reason':snap.get('transcode_reason') or '',
|
| 601 |
}
|
| 602 |
|
| 603 |
def _cgroup_usage_ns():
|
|
|
|
| 822 |
'last_error':tsk.get('last_error') or '','source_age':rs.get('source_age'),
|
| 823 |
'output_seconds':rs.get('output_seconds') or 0,
|
| 824 |
'send_mode':rs.get('send_mode') or '',
|
| 825 |
+
'relay_state':rs.get('relay_state') or '',
|
| 826 |
+
'transcode_reason':rs.get('transcode_reason') or '',
|
| 827 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 828 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 829 |
'youtube_status':str(ye.get('youtube_status') or '')})
|
|
|
|
| 938 |
'last_data_time':rs.get('last_data_time'),'relay_alive':rs.get('relay_alive'),
|
| 939 |
'session_title':rs.get('session_title') or '','restarts':rs.get('restarts') or 0,
|
| 940 |
'source_age':rs.get('source_age'),
|
| 941 |
+
'send_mode':rs.get('send_mode') or '',
|
| 942 |
+
'relay_state':rs.get('relay_state') or '',
|
| 943 |
+
'transcode_reason':rs.get('transcode_reason') or '',
|
| 944 |
+
'output_seconds':rs.get('output_seconds') or 0,
|
| 945 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 946 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 947 |
'youtube_status':str(ye.get('youtube_status') or ('relay' if rs.get('relay_alive') else (t.get('state') or '')))})
|
index.html
CHANGED
|
@@ -144,8 +144,9 @@ async function refresh(){
|
|
| 144 |
${x.nickname?' · '+esc(x.nickname):''}</p>
|
| 145 |
<p>LIVE منذ: ${live?esc(fmtDur(x.live_for||0)):'—'}</p>
|
| 146 |
<p>المصدر: ${esc(x.source||'—')}</p>
|
|
|
|
| 147 |
<p>Codec: ${esc(x.codec||'—')} · ${esc(x.quality||'')} · room ${esc(x.room_id||'—')}</p>
|
| 148 |
-
<p>PID: ${x.relay_pid||'—'} · بايت: ${esc(x.bytes_received||0)} · إعادة: ${esc(x.restarts||0)}</p>
|
| 149 |
<p>آخر بايت: ${x.source_age==null||x.source_age===''?'—':(Number(x.source_age).toFixed(1)+'ث')}</p>
|
| 150 |
<p>YouTube: ${esc(x.youtube_status||'—')} · health ${esc(x.youtube_health||'—')}${x.relay_alive?' · Relay':''}</p>
|
| 151 |
<small>${esc(x.status||'')}${x.last_error?' · '+esc(x.last_error):''}</small>
|
|
|
|
| 144 |
${x.nickname?' · '+esc(x.nickname):''}</p>
|
| 145 |
<p>LIVE منذ: ${live?esc(fmtDur(x.live_for||0)):'—'}</p>
|
| 146 |
<p>المصدر: ${esc(x.source||'—')}</p>
|
| 147 |
+
<p>MODE: ${esc(x.send_mode||x.relay_state||'—')}${x.transcode_reason?' · '+esc(x.transcode_reason):''}</p>
|
| 148 |
<p>Codec: ${esc(x.codec||'—')} · ${esc(x.quality||'')} · room ${esc(x.room_id||'—')}</p>
|
| 149 |
+
<p>PID: ${x.relay_pid||'—'} · بايت: ${esc(x.bytes_received||0)} · إعادة: ${esc(x.restarts||0)} · out ${esc(x.output_seconds||0)}ث</p>
|
| 150 |
<p>آخر بايت: ${x.source_age==null||x.source_age===''?'—':(Number(x.source_age).toFixed(1)+'ث')}</p>
|
| 151 |
<p>YouTube: ${esc(x.youtube_status||'—')} · health ${esc(x.youtube_health||'—')}${x.relay_alive?' · Relay':''}</p>
|
| 152 |
<small>${esc(x.status||'')}${x.last_error?' · '+esc(x.last_error):''}</small>
|
relay.py
CHANGED
|
@@ -100,6 +100,13 @@ class Sender:
|
|
| 100 |
# event. Without this the flap window expires after 10 quiet minutes and the
|
| 101 |
# sender silently drops back to copy mode, restarting the whole failure loop.
|
| 102 |
self.sticky_mode=None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
# Candidate handed to send_mode() so the codec rule can judge the real source.
|
| 104 |
self.current_candidate=None
|
| 105 |
# Journal setting wins over the environment so the mode can be changed from the UI
|
|
@@ -117,9 +124,11 @@ class Sender:
|
|
| 117 |
stamp=d.get('last_source_at')
|
| 118 |
if stamp:d['source_age']=round(max(0.0,time.monotonic()-stamp),1)
|
| 119 |
try:
|
| 120 |
-
d['send_mode']=self.sticky_mode or self.mode_override or ''
|
| 121 |
except Exception:
|
| 122 |
d['send_mode']=''
|
|
|
|
|
|
|
| 123 |
# Hide leftover TLS/EOF from a previous handshake while this FFmpeg is actually sending.
|
| 124 |
try:
|
| 125 |
alive=self.alive()
|
|
@@ -173,7 +182,8 @@ class Sender:
|
|
| 173 |
# two fixed720 publishers used 1.67–1.85 of 2.0 quota (peak 92.6%) with YouTube
|
| 174 |
# health still good — no headroom for a third. If this candidate is H264, copy.
|
| 175 |
if self.sticky_mode:
|
| 176 |
-
# light
|
|
|
|
| 177 |
if self.sticky_mode=='light':
|
| 178 |
return 'light'
|
| 179 |
cand=self.current_candidate or {}
|
|
@@ -338,6 +348,14 @@ class Sender:
|
|
| 338 |
def command(self,destination,candidate=None):
|
| 339 |
# إعلان Enhanced-RTMP يُقاس على الترميز المدفوع فعلًا؛ ملفات الاختبار المحلية لا بروتوكول لها.
|
| 340 |
mode=self.send_mode()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 341 |
network=destination.startswith(('rtmp://','rtmps://'))
|
| 342 |
enhanced=self.enhanced_advertisement(mode,candidate) if network else []
|
| 343 |
if mode=='copy':
|
|
|
|
| 100 |
# event. Without this the flap window expires after 10 quiet minutes and the
|
| 101 |
# sender silently drops back to copy mode, restarting the whole failure loop.
|
| 102 |
self.sticky_mode=None
|
| 103 |
+
self.last_send_mode=''
|
| 104 |
+
self.relay_state='IDLE'
|
| 105 |
+
self.transcode_reason=''
|
| 106 |
+
self.transcode_since=None
|
| 107 |
+
self.copy_ok_since=None
|
| 108 |
+
self.yt_health=''
|
| 109 |
+
self.session_out_codec='h264'
|
| 110 |
# Candidate handed to send_mode() so the codec rule can judge the real source.
|
| 111 |
self.current_candidate=None
|
| 112 |
# Journal setting wins over the environment so the mode can be changed from the UI
|
|
|
|
| 124 |
stamp=d.get('last_source_at')
|
| 125 |
if stamp:d['source_age']=round(max(0.0,time.monotonic()-stamp),1)
|
| 126 |
try:
|
| 127 |
+
d['send_mode']=self.last_send_mode or self.sticky_mode or self.mode_override or ''
|
| 128 |
except Exception:
|
| 129 |
d['send_mode']=''
|
| 130 |
+
d['relay_state']=getattr(self,'relay_state','') or ''
|
| 131 |
+
d['transcode_reason']=getattr(self,'transcode_reason','') or ''
|
| 132 |
# Hide leftover TLS/EOF from a previous handshake while this FFmpeg is actually sending.
|
| 133 |
try:
|
| 134 |
alive=self.alive()
|
|
|
|
| 182 |
# two fixed720 publishers used 1.67–1.85 of 2.0 quota (peak 92.6%) with YouTube
|
| 183 |
# health still good — no headroom for a third. If this candidate is H264, copy.
|
| 184 |
if self.sticky_mode:
|
| 185 |
+
# light stays until the manager hysteresis (H264 + YT health good for
|
| 186 |
+
# YOUTUBE_COPY_RECOVERY_SECONDS). Do not drop on the first H264 packet.
|
| 187 |
if self.sticky_mode=='light':
|
| 188 |
return 'light'
|
| 189 |
cand=self.current_candidate or {}
|
|
|
|
| 348 |
def command(self,destination,candidate=None):
|
| 349 |
# إعلان Enhanced-RTMP يُقاس على الترميز المدفوع فعلًا؛ ملفات الاختبار المحلية لا بروتوكول لها.
|
| 350 |
mode=self.send_mode()
|
| 351 |
+
self.last_send_mode=mode
|
| 352 |
+
if mode=='copy':
|
| 353 |
+
if self.relay_state in ('IDLE','COPY_RECOVERY','REATTACH','ALTERNATE_SOURCE','SOURCE_FAULT',''):
|
| 354 |
+
self.relay_state='COPY_STABLE'
|
| 355 |
+
elif mode=='light':
|
| 356 |
+
self.relay_state='TRANSCODE_STABLE' if self.relay_state in ('TRANSCODE_FALLBACK','TRANSCODE_STABLE') else 'TRANSCODE_FALLBACK'
|
| 357 |
+
if self.transcode_since is None:
|
| 358 |
+
self.transcode_since=time.monotonic()
|
| 359 |
network=destination.startswith(('rtmp://','rtmps://'))
|
| 360 |
enhanced=self.enhanced_advertisement(mode,candidate) if network else []
|
| 361 |
if mode=='copy':
|
youtube_manager.py
CHANGED
|
@@ -1767,6 +1767,39 @@ class ManagedYouTube:
|
|
| 1767 |
self.event_note(key,'انتظار تأكيد استقبال يوتيوب قبل بدء الحدث')
|
| 1768 |
|
| 1769 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1770 |
def _light_count(self, except_sender=None):
|
| 1771 |
n=0
|
| 1772 |
for e in (self.senders or {}).values():
|
|
@@ -1788,6 +1821,10 @@ class ManagedYouTube:
|
|
| 1788 |
self.log(user,'[RELAY] ترميز خفيف عند السقف (%d)؛ نُبقي copy'%cap)
|
| 1789 |
return False
|
| 1790 |
sender.sticky_mode='light'
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1791 |
if key:
|
| 1792 |
self.event_update(key, force_light=True)
|
| 1793 |
self.log(user,'[RELAY] copy لم يحقق ingest مستقر (%s) — ترميز خفيف ultrafast 1800k GOP 2ث'%why)
|
|
@@ -1801,6 +1838,11 @@ class ManagedYouTube:
|
|
| 1801 |
health=(ev.get('youtube_health') or '')
|
| 1802 |
if health in ('good','ok'):
|
| 1803 |
self._ingest_bad_since.pop(key,None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1804 |
return
|
| 1805 |
if health not in ('noData','bad','noStream'):
|
| 1806 |
return
|
|
@@ -1869,11 +1911,16 @@ class ManagedYouTube:
|
|
| 1869 |
# GOP too long / starved ingest → YouTube loading. bitrateLow alone is not enough
|
| 1870 |
# (measured: 3× H264 copy ~1 Mbps stayed good/ok at ~8% CPU).
|
| 1871 |
hard=any(k in (text or '') for k in ('gopSizeLong','openGop','videoIngestionStarved'))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1872 |
if hard:
|
| 1873 |
-
entry=self.senders.get(key) or {}
|
| 1874 |
-
snd=entry.get('sender')
|
| 1875 |
if snd and self._maybe_light(user, snd, text[:80], key=key):
|
| 1876 |
self.stop_sender(key)
|
|
|
|
|
|
|
|
|
|
| 1877 |
def run_manager(self):
|
| 1878 |
checked=False
|
| 1879 |
last_tick=0.0
|
|
|
|
| 1767 |
self.event_note(key,'انتظار تأكيد استقبال يوتيوب قبل بدء الحدث')
|
| 1768 |
|
| 1769 |
|
| 1770 |
+
|
| 1771 |
+
def _maybe_copy_recovery(self, key, user, ev, snd):
|
| 1772 |
+
"""Hysteresis: light → copy only after H264 + YT health good for recovery window.
|
| 1773 |
+
|
| 1774 |
+
First H264 packet is not enough (oscillation). Window default 45s = one fast
|
| 1775 |
+
YouTube health cycle (measured YOUTUBE_HEALTH_FAST_INTERVAL).
|
| 1776 |
+
"""
|
| 1777 |
+
if not snd or getattr(snd,'sticky_mode',None)!='light':
|
| 1778 |
+
if snd: snd.copy_ok_since=None
|
| 1779 |
+
return False
|
| 1780 |
+
health=(ev.get('youtube_health') or getattr(snd,'yt_health','') or '')
|
| 1781 |
+
cand=snd.current_candidate or {}
|
| 1782 |
+
blob=(str(cand.get('codec') or '')+' '+str(cand.get('label') or '')+' '+str(cand.get('brief') or '')).lower()
|
| 1783 |
+
h264=any(k in blob for k in ('h264','avc'))
|
| 1784 |
+
if health not in ('good','ok') or not h264:
|
| 1785 |
+
snd.copy_ok_since=None
|
| 1786 |
+
return False
|
| 1787 |
+
now=time.monotonic()
|
| 1788 |
+
if snd.copy_ok_since is None:
|
| 1789 |
+
snd.copy_ok_since=now
|
| 1790 |
+
snd.relay_state='TRANSCODE_STABLE'
|
| 1791 |
+
return False
|
| 1792 |
+
need=float(os.getenv('YOUTUBE_COPY_RECOVERY_SECONDS','45'))
|
| 1793 |
+
if now-snd.copy_ok_since<need:
|
| 1794 |
+
return False
|
| 1795 |
+
snd.sticky_mode=None
|
| 1796 |
+
snd.relay_state='COPY_RECOVERY'
|
| 1797 |
+
snd.transcode_reason=''
|
| 1798 |
+
self.event_update(key, force_light=False)
|
| 1799 |
+
self.log(user,'[RELAY] H264 مستقر %.0fs وyoutube_health=%s — عودة copy على نفس الحدث'%(need,health))
|
| 1800 |
+
self.stop_sender(key)
|
| 1801 |
+
return True
|
| 1802 |
+
|
| 1803 |
def _light_count(self, except_sender=None):
|
| 1804 |
n=0
|
| 1805 |
for e in (self.senders or {}).values():
|
|
|
|
| 1821 |
self.log(user,'[RELAY] ترميز خفيف عند السقف (%d)؛ نُبقي copy'%cap)
|
| 1822 |
return False
|
| 1823 |
sender.sticky_mode='light'
|
| 1824 |
+
sender.relay_state='TRANSCODE_FALLBACK'
|
| 1825 |
+
sender.transcode_reason=str(why or '')[:160]
|
| 1826 |
+
sender.transcode_since=time.monotonic()
|
| 1827 |
+
sender.copy_ok_since=None
|
| 1828 |
if key:
|
| 1829 |
self.event_update(key, force_light=True)
|
| 1830 |
self.log(user,'[RELAY] copy لم يحقق ingest مستقر (%s) — ترميز خفيف ultrafast 1800k GOP 2ث'%why)
|
|
|
|
| 1838 |
health=(ev.get('youtube_health') or '')
|
| 1839 |
if health in ('good','ok'):
|
| 1840 |
self._ingest_bad_since.pop(key,None)
|
| 1841 |
+
entry=self.senders.get(key) or {}
|
| 1842 |
+
snd=entry.get('sender')
|
| 1843 |
+
if snd:
|
| 1844 |
+
snd.yt_health=health
|
| 1845 |
+
self._maybe_copy_recovery(key,user,ev,snd)
|
| 1846 |
return
|
| 1847 |
if health not in ('noData','bad','noStream'):
|
| 1848 |
return
|
|
|
|
| 1911 |
# GOP too long / starved ingest → YouTube loading. bitrateLow alone is not enough
|
| 1912 |
# (measured: 3× H264 copy ~1 Mbps stayed good/ok at ~8% CPU).
|
| 1913 |
hard=any(k in (text or '') for k in ('gopSizeLong','openGop','videoIngestionStarved'))
|
| 1914 |
+
entry=self.senders.get(key) or {}
|
| 1915 |
+
snd=entry.get('sender')
|
| 1916 |
+
if snd:
|
| 1917 |
+
snd.yt_health=health.get('status','')
|
| 1918 |
if hard:
|
|
|
|
|
|
|
| 1919 |
if snd and self._maybe_light(user, snd, text[:80], key=key):
|
| 1920 |
self.stop_sender(key)
|
| 1921 |
+
return
|
| 1922 |
+
if snd:
|
| 1923 |
+
self._maybe_copy_recovery(key,user,self.journal.read()['events'].get(key) or ev,snd)
|
| 1924 |
def run_manager(self):
|
| 1925 |
checked=False
|
| 1926 |
last_tick=0.0
|