On-only test: copy-first + light fallback, MAX=4, freeze/liveStarting fixes
Browse files- Dockerfile +7 -4
- app.py +8 -0
- index.html +2 -1
- relay.py +65 -16
- youtube_manager.py +196 -8
Dockerfile
CHANGED
|
@@ -13,24 +13,27 @@ ENV TZ=Asia/Baghdad \
|
|
| 13 |
CHECK_INTERVAL=20 \
|
| 14 |
SOURCE_STALL_TIMEOUT=12 \
|
| 15 |
YOUTUBE_SOURCE_STALL_SECONDS=12 \
|
| 16 |
-
MAX_CONCURRENT_RELAYS=
|
| 17 |
-
YOUTUBE_MAX_EVENTS=
|
| 18 |
YOUTUBE_TITLE_TEMPLATE="{broadcaster} | {date} {time}" \
|
| 19 |
YOUTUBE_SEND_MODE=auto \
|
| 20 |
YOUTUBE_AUTO_TRANSCODE=1 \
|
|
|
|
|
|
|
|
|
|
| 21 |
YOUTUBE_AUDIO_PROFILE=aac_low \
|
| 22 |
YOUTUBE_AUDIO_BITRATE=128k \
|
| 23 |
YOUTUBE_AUDIO_CHANNELS=2 \
|
| 24 |
YOUTUBE_GOP=60 \
|
| 25 |
YOUTUBE_INPUT_RECOVERY=1 \
|
| 26 |
YOUTUBE_PREFER_H264=1 \
|
| 27 |
-
YOUTUBE_CODEC_FALLBACK_MODE=
|
| 28 |
YOUTUBE_RETRY_MIN=1 \
|
| 29 |
YOUTUBE_RETRY_MAX=60 \
|
| 30 |
YOUTUBE_SOURCE_LIMIT=6 \
|
| 31 |
YOUTUBE_FORMAT_GUARD=on \
|
| 32 |
YOUTUBE_FLAP_LIMIT=4 \
|
| 33 |
-
YOUTUBE_FLAP_FALLBACK_MODE=
|
| 34 |
YOUTUBE_X264_PRESET=veryfast \
|
| 35 |
YOUTUBE_CRF=22 \
|
| 36 |
YOUTUBE_FIFO_TIMESHIFT=0 \
|
|
|
|
| 13 |
CHECK_INTERVAL=20 \
|
| 14 |
SOURCE_STALL_TIMEOUT=12 \
|
| 15 |
YOUTUBE_SOURCE_STALL_SECONDS=12 \
|
| 16 |
+
MAX_CONCURRENT_RELAYS=4 \
|
| 17 |
+
YOUTUBE_MAX_EVENTS=4 \
|
| 18 |
YOUTUBE_TITLE_TEMPLATE="{broadcaster} | {date} {time}" \
|
| 19 |
YOUTUBE_SEND_MODE=auto \
|
| 20 |
YOUTUBE_AUTO_TRANSCODE=1 \
|
| 21 |
+
YOUTUBE_LIGHT_MAX=1 \
|
| 22 |
+
YOUTUBE_LIGHT_BITRATE=1800k \
|
| 23 |
+
YOUTUBE_COPY_RECOVERY_SECONDS=45 \
|
| 24 |
YOUTUBE_AUDIO_PROFILE=aac_low \
|
| 25 |
YOUTUBE_AUDIO_BITRATE=128k \
|
| 26 |
YOUTUBE_AUDIO_CHANNELS=2 \
|
| 27 |
YOUTUBE_GOP=60 \
|
| 28 |
YOUTUBE_INPUT_RECOVERY=1 \
|
| 29 |
YOUTUBE_PREFER_H264=1 \
|
| 30 |
+
YOUTUBE_CODEC_FALLBACK_MODE=light \
|
| 31 |
YOUTUBE_RETRY_MIN=1 \
|
| 32 |
YOUTUBE_RETRY_MAX=60 \
|
| 33 |
YOUTUBE_SOURCE_LIMIT=6 \
|
| 34 |
YOUTUBE_FORMAT_GUARD=on \
|
| 35 |
YOUTUBE_FLAP_LIMIT=4 \
|
| 36 |
+
YOUTUBE_FLAP_FALLBACK_MODE=light \
|
| 37 |
YOUTUBE_X264_PRESET=veryfast \
|
| 38 |
YOUTUBE_CRF=22 \
|
| 39 |
YOUTUBE_FIFO_TIMESHIFT=0 \
|
app.py
CHANGED
|
@@ -596,6 +596,8 @@ def _relay_snapshot(user):
|
|
| 596 |
'relay_dest':slot.get('dest') if slot else '',
|
| 597 |
'restarts':hub.restart_count(user),
|
| 598 |
'send_mode':snap.get('send_mode') or '',
|
|
|
|
|
|
|
| 599 |
}
|
| 600 |
|
| 601 |
def _cgroup_usage_ns():
|
|
@@ -820,6 +822,8 @@ def health_public():
|
|
| 820 |
'last_error':tsk.get('last_error') or '','source_age':rs.get('source_age'),
|
| 821 |
'output_seconds':rs.get('output_seconds') or 0,
|
| 822 |
'send_mode':rs.get('send_mode') or '',
|
|
|
|
|
|
|
| 823 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 824 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 825 |
'youtube_status':str(ye.get('youtube_status') or '')})
|
|
@@ -934,6 +938,10 @@ def state():
|
|
| 934 |
'last_data_time':rs.get('last_data_time'),'relay_alive':rs.get('relay_alive'),
|
| 935 |
'session_title':rs.get('session_title') or '','restarts':rs.get('restarts') or 0,
|
| 936 |
'source_age':rs.get('source_age'),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 937 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 938 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 939 |
'youtube_status':str(ye.get('youtube_status') or ('relay' if rs.get('relay_alive') else (t.get('state') or '')))})
|
|
|
|
| 596 |
'relay_dest':slot.get('dest') if slot else '',
|
| 597 |
'restarts':hub.restart_count(user),
|
| 598 |
'send_mode':snap.get('send_mode') or '',
|
| 599 |
+
'relay_state':snap.get('relay_state') or '',
|
| 600 |
+
'transcode_reason':snap.get('transcode_reason') or '',
|
| 601 |
}
|
| 602 |
|
| 603 |
def _cgroup_usage_ns():
|
|
|
|
| 822 |
'last_error':tsk.get('last_error') or '','source_age':rs.get('source_age'),
|
| 823 |
'output_seconds':rs.get('output_seconds') or 0,
|
| 824 |
'send_mode':rs.get('send_mode') or '',
|
| 825 |
+
'relay_state':rs.get('relay_state') or '',
|
| 826 |
+
'transcode_reason':rs.get('transcode_reason') or '',
|
| 827 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 828 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 829 |
'youtube_status':str(ye.get('youtube_status') or '')})
|
|
|
|
| 938 |
'last_data_time':rs.get('last_data_time'),'relay_alive':rs.get('relay_alive'),
|
| 939 |
'session_title':rs.get('session_title') or '','restarts':rs.get('restarts') or 0,
|
| 940 |
'source_age':rs.get('source_age'),
|
| 941 |
+
'send_mode':rs.get('send_mode') or '',
|
| 942 |
+
'relay_state':rs.get('relay_state') or '',
|
| 943 |
+
'transcode_reason':rs.get('transcode_reason') or '',
|
| 944 |
+
'output_seconds':rs.get('output_seconds') or 0,
|
| 945 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 946 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 947 |
'youtube_status':str(ye.get('youtube_status') or ('relay' if rs.get('relay_alive') else (t.get('state') or '')))})
|
index.html
CHANGED
|
@@ -144,8 +144,9 @@ async function refresh(){
|
|
| 144 |
${x.nickname?' · '+esc(x.nickname):''}</p>
|
| 145 |
<p>LIVE منذ: ${live?esc(fmtDur(x.live_for||0)):'—'}</p>
|
| 146 |
<p>المصدر: ${esc(x.source||'—')}</p>
|
|
|
|
| 147 |
<p>Codec: ${esc(x.codec||'—')} · ${esc(x.quality||'')} · room ${esc(x.room_id||'—')}</p>
|
| 148 |
-
<p>PID: ${x.relay_pid||'—'} · بايت: ${esc(x.bytes_received||0)} · إعادة: ${esc(x.restarts||0)}</p>
|
| 149 |
<p>آخر بايت: ${x.source_age==null||x.source_age===''?'—':(Number(x.source_age).toFixed(1)+'ث')}</p>
|
| 150 |
<p>YouTube: ${esc(x.youtube_status||'—')} · health ${esc(x.youtube_health||'—')}${x.relay_alive?' · Relay':''}</p>
|
| 151 |
<small>${esc(x.status||'')}${x.last_error?' · '+esc(x.last_error):''}</small>
|
|
|
|
| 144 |
${x.nickname?' · '+esc(x.nickname):''}</p>
|
| 145 |
<p>LIVE منذ: ${live?esc(fmtDur(x.live_for||0)):'—'}</p>
|
| 146 |
<p>المصدر: ${esc(x.source||'—')}</p>
|
| 147 |
+
<p>MODE: ${esc(x.send_mode||x.relay_state||'—')}${x.transcode_reason?' · '+esc(x.transcode_reason):''}</p>
|
| 148 |
<p>Codec: ${esc(x.codec||'—')} · ${esc(x.quality||'')} · room ${esc(x.room_id||'—')}</p>
|
| 149 |
+
<p>PID: ${x.relay_pid||'—'} · بايت: ${esc(x.bytes_received||0)} · إعادة: ${esc(x.restarts||0)} · out ${esc(x.output_seconds||0)}ث</p>
|
| 150 |
<p>آخر بايت: ${x.source_age==null||x.source_age===''?'—':(Number(x.source_age).toFixed(1)+'ث')}</p>
|
| 151 |
<p>YouTube: ${esc(x.youtube_status||'—')} · health ${esc(x.youtube_health||'—')}${x.relay_alive?' · Relay':''}</p>
|
| 152 |
<small>${esc(x.status||'')}${x.last_error?' · '+esc(x.last_error):''}</small>
|
relay.py
CHANGED
|
@@ -100,6 +100,13 @@ class Sender:
|
|
| 100 |
# event. Without this the flap window expires after 10 quiet minutes and the
|
| 101 |
# sender silently drops back to copy mode, restarting the whole failure loop.
|
| 102 |
self.sticky_mode=None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
# Candidate handed to send_mode() so the codec rule can judge the real source.
|
| 104 |
self.current_candidate=None
|
| 105 |
# Journal setting wins over the environment so the mode can be changed from the UI
|
|
@@ -117,9 +124,11 @@ class Sender:
|
|
| 117 |
stamp=d.get('last_source_at')
|
| 118 |
if stamp:d['source_age']=round(max(0.0,time.monotonic()-stamp),1)
|
| 119 |
try:
|
| 120 |
-
d['send_mode']=self.sticky_mode or self.mode_override or ''
|
| 121 |
except Exception:
|
| 122 |
d['send_mode']=''
|
|
|
|
|
|
|
| 123 |
# Hide leftover TLS/EOF from a previous handshake while this FFmpeg is actually sending.
|
| 124 |
try:
|
| 125 |
alive=self.alive()
|
|
@@ -167,12 +176,16 @@ class Sender:
|
|
| 167 |
fixed720 -> constant 720x1280 canvas, re-encodes (lighter on CPU)
|
| 168 |
"""
|
| 169 |
mode=(self.mode_override or os.getenv('YOUTUBE_SEND_MODE','auto')).strip().lower()
|
| 170 |
-
if mode not in ('auto','copy','fixed1080','fixed720'):
|
| 171 |
-
raise ValueError('وضع الإرسال يجب أن يكون auto أو copy أو fixed1080 أو fixed720')
|
| 172 |
# Sticky transcode is for HEVC / layout flap only. Measured 2026-09-27 on cpu-basic:
|
| 173 |
# two fixed720 publishers used 1.67–1.85 of 2.0 quota (peak 92.6%) with YouTube
|
| 174 |
# health still good — no headroom for a third. If this candidate is H264, copy.
|
| 175 |
if self.sticky_mode:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
cand=self.current_candidate or {}
|
| 177 |
blob=(str(cand.get('codec') or '')+' '+str(cand.get('label') or '')+' '+str(cand.get('brief') or '')).lower()
|
| 178 |
if any(k in blob for k in ('h264','avc')):
|
|
@@ -195,18 +208,17 @@ class Sender:
|
|
| 195 |
now=time.monotonic()
|
| 196 |
while flap and now-flap[0]>600:flap.popleft()
|
| 197 |
if len(flap)>=int(os.getenv('YOUTUBE_FLAP_LIMIT','4')):
|
| 198 |
-
fb=os.getenv('YOUTUBE_FLAP_FALLBACK_MODE','').strip().lower()
|
| 199 |
-
|
| 200 |
-
if fb in ('fixed1080','fixed720'):
|
| 201 |
self.sticky_mode=fb
|
| 202 |
mode=fb
|
| 203 |
return mode
|
| 204 |
def note_format_change(self):
|
| 205 |
self.flap.append(time.monotonic())
|
| 206 |
if len(self.flap)>=int(os.getenv('YOUTUBE_FLAP_LIMIT','4')):
|
| 207 |
-
fb=os.getenv('YOUTUBE_FLAP_FALLBACK_MODE','').strip().lower()
|
| 208 |
-
if fb in ('fixed1080','fixed720'):
|
| 209 |
-
self.log('system','تغيّرت أبعاد المصدر عدة مرات
|
| 210 |
else:
|
| 211 |
self.log('system','تغيّرت أبعاد المصدر عدة مرات؛ نُبقي copy ونعيد توصيل تيك توك على نفس جلسة يوتيوب')
|
| 212 |
@staticmethod
|
|
@@ -244,12 +256,11 @@ class Sender:
|
|
| 244 |
label=str(cand.get('label') or cand.get('brief') or '')
|
| 245 |
if any(k in str(declared).lower() for k in ('h264','avc')) or any(k in label.lower() for k in ('h264','avc')):
|
| 246 |
return None
|
| 247 |
-
mode=os.getenv('YOUTUBE_CODEC_FALLBACK_MODE','
|
| 248 |
-
if mode not in ('fixed1080','fixed720'):mode='
|
| 249 |
self.sticky_mode=mode
|
| 250 |
why=('ffprobe: '+str(probed)) if probed else (('مُعلَن: '+declared) if declared else 'ترميز مجهول والفحص لم يأتِ بنتيجة')
|
| 251 |
-
self.log('system','لا يمكن نسخ المصدر كما هو ('+why+') — ت
|
| 252 |
-
'؛ هذا يمنع تجمّد الصورة عند المشاهدين عند تغيّر المقاس أو في جولات PK')
|
| 253 |
return mode
|
| 254 |
def set_probe(self,probe):
|
| 255 |
"""Injected by the manager: probe(candidate) -> {'video':..,'audio':..} or None."""
|
|
@@ -337,14 +348,34 @@ class Sender:
|
|
| 337 |
def command(self,destination,candidate=None):
|
| 338 |
# إعلان Enhanced-RTMP يُقاس على الترميز المدفوع فعلًا؛ ملفات الاختبار المحلية لا بروتوكول لها.
|
| 339 |
mode=self.send_mode()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
network=destination.startswith(('rtmp://','rtmps://'))
|
| 341 |
enhanced=self.enhanced_advertisement(mode,candidate) if network else []
|
| 342 |
if mode=='copy':
|
| 343 |
video=['-c:v','copy']
|
| 344 |
-
# Repeat SPS/PPS on every IDR so a YouTube viewer who hits loading can
|
| 345 |
-
# resync without leaving the app. Video remains copy (no re-encode).
|
| 346 |
if not self.hevc_payload(candidate):
|
| 347 |
video += ['-bsf:v','dump_extra=freq=keyframe']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 348 |
else:
|
| 349 |
width,height,bitrate=(1080,1920,'6000k') if mode=='fixed1080' else (720,1280,'4000k')
|
| 350 |
# VIDEO_BITRATE was documented in the README but never read anywhere; it now caps the
|
|
@@ -627,7 +658,8 @@ class Sender:
|
|
| 627 |
sess['stall_reason'] = ''
|
| 628 |
sess['force_stop'] = False
|
| 629 |
sess['timeouts'] = 0
|
| 630 |
-
sess['last']
|
|
|
|
| 631 |
sess['reattaches'] = int(sess.get('reattaches') or 0) + 1
|
| 632 |
self.current_candidate = cand
|
| 633 |
self.set(source=cand.get('label') or cand.get('brief') or '', stalled=False)
|
|
@@ -822,15 +854,32 @@ class Sender:
|
|
| 822 |
# ---- الحارس: لا تعتمد على poll() ولا على رمز الخروج ----
|
| 823 |
stall_limit = self._stall_seconds()
|
| 824 |
progress_limit = max(30.0, float(os.getenv('YOUTUBE_PROGRESS_STALL_SECONDS', '60')))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 825 |
next_report = 0.0
|
| 826 |
while not stop.is_set():
|
| 827 |
now = time.monotonic()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 828 |
# الدليل على أن المصدر لم يصمت هو بايتات تيك توك نفسها. تقدّم FFmpeg وحده لا
|
| 829 |
# يكفي: في وضع النسخ يفرّغ المخزن الداخلي لدقائق بعد صمت المصدر، فيبدو البث
|
| 830 |
# حيًا ويوتيوب يبقى على آخر ثانية — وهذا هو التعليق الذي شكا منه المستخدم.
|
| 831 |
# في HLS يستحيل القياس إلا على FFmpeg (هو من يفتح القائمة)، فنقبل تقدّمه.
|
| 832 |
_fresh = (sess['ff_last'] if response is None else sess['last'])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 833 |
if stall_limit and not sess.get('swapping') and now - _fresh > stall_limit:
|
|
|
|
|
|
|
| 834 |
attached=None
|
| 835 |
for _try in range(3):
|
| 836 |
attached = self._try_reattach(user, p, bridge, sess, stop, refresh,
|
|
|
|
| 100 |
# event. Without this the flap window expires after 10 quiet minutes and the
|
| 101 |
# sender silently drops back to copy mode, restarting the whole failure loop.
|
| 102 |
self.sticky_mode=None
|
| 103 |
+
self.last_send_mode=''
|
| 104 |
+
self.relay_state='IDLE'
|
| 105 |
+
self.transcode_reason=''
|
| 106 |
+
self.transcode_since=None
|
| 107 |
+
self.copy_ok_since=None
|
| 108 |
+
self.yt_health=''
|
| 109 |
+
self.session_out_codec='h264'
|
| 110 |
# Candidate handed to send_mode() so the codec rule can judge the real source.
|
| 111 |
self.current_candidate=None
|
| 112 |
# Journal setting wins over the environment so the mode can be changed from the UI
|
|
|
|
| 124 |
stamp=d.get('last_source_at')
|
| 125 |
if stamp:d['source_age']=round(max(0.0,time.monotonic()-stamp),1)
|
| 126 |
try:
|
| 127 |
+
d['send_mode']=self.last_send_mode or self.sticky_mode or self.mode_override or ''
|
| 128 |
except Exception:
|
| 129 |
d['send_mode']=''
|
| 130 |
+
d['relay_state']=getattr(self,'relay_state','') or ''
|
| 131 |
+
d['transcode_reason']=getattr(self,'transcode_reason','') or ''
|
| 132 |
# Hide leftover TLS/EOF from a previous handshake while this FFmpeg is actually sending.
|
| 133 |
try:
|
| 134 |
alive=self.alive()
|
|
|
|
| 176 |
fixed720 -> constant 720x1280 canvas, re-encodes (lighter on CPU)
|
| 177 |
"""
|
| 178 |
mode=(self.mode_override or os.getenv('YOUTUBE_SEND_MODE','auto')).strip().lower()
|
| 179 |
+
if mode not in ('auto','copy','light','fixed1080','fixed720'):
|
| 180 |
+
raise ValueError('وضع الإرسال يجب أن يكون auto أو copy أو light أو fixed1080 أو fixed720')
|
| 181 |
# Sticky transcode is for HEVC / layout flap only. Measured 2026-09-27 on cpu-basic:
|
| 182 |
# two fixed720 publishers used 1.67–1.85 of 2.0 quota (peak 92.6%) with YouTube
|
| 183 |
# health still good — no headroom for a third. If this candidate is H264, copy.
|
| 184 |
if self.sticky_mode:
|
| 185 |
+
# light stays until the manager hysteresis (H264 + YT health good for
|
| 186 |
+
# YOUTUBE_COPY_RECOVERY_SECONDS). Do not drop on the first H264 packet.
|
| 187 |
+
if self.sticky_mode=='light':
|
| 188 |
+
return 'light'
|
| 189 |
cand=self.current_candidate or {}
|
| 190 |
blob=(str(cand.get('codec') or '')+' '+str(cand.get('label') or '')+' '+str(cand.get('brief') or '')).lower()
|
| 191 |
if any(k in blob for k in ('h264','avc')):
|
|
|
|
| 208 |
now=time.monotonic()
|
| 209 |
while flap and now-flap[0]>600:flap.popleft()
|
| 210 |
if len(flap)>=int(os.getenv('YOUTUBE_FLAP_LIMIT','4')):
|
| 211 |
+
fb=os.getenv('YOUTUBE_FLAP_FALLBACK_MODE','light').strip().lower()
|
| 212 |
+
if fb in ('light','fixed1080','fixed720'):
|
|
|
|
| 213 |
self.sticky_mode=fb
|
| 214 |
mode=fb
|
| 215 |
return mode
|
| 216 |
def note_format_change(self):
|
| 217 |
self.flap.append(time.monotonic())
|
| 218 |
if len(self.flap)>=int(os.getenv('YOUTUBE_FLAP_LIMIT','4')):
|
| 219 |
+
fb=os.getenv('YOUTUBE_FLAP_FALLBACK_MODE','light').strip().lower()
|
| 220 |
+
if fb in ('light','fixed1080','fixed720'):
|
| 221 |
+
self.log('system','تغيّرت أبعاد المصدر عدة مرات؛ copy لم يكفِ — ترميز خفيف (%s) بمفتاح IDR كل 2ث'%fb)
|
| 222 |
else:
|
| 223 |
self.log('system','تغيّرت أبعاد المصدر عدة مرات؛ نُبقي copy ونعيد توصيل تيك توك على نفس جلسة يوتيوب')
|
| 224 |
@staticmethod
|
|
|
|
| 256 |
label=str(cand.get('label') or cand.get('brief') or '')
|
| 257 |
if any(k in str(declared).lower() for k in ('h264','avc')) or any(k in label.lower() for k in ('h264','avc')):
|
| 258 |
return None
|
| 259 |
+
mode=os.getenv('YOUTUBE_CODEC_FALLBACK_MODE','light').strip().lower()
|
| 260 |
+
if mode not in ('light','fixed1080','fixed720'):mode='light'
|
| 261 |
self.sticky_mode=mode
|
| 262 |
why=('ffprobe: '+str(probed)) if probed else (('مُعلَن: '+declared) if declared else 'ترميز مجهول والفحص لم يأتِ بنتيجة')
|
| 263 |
+
self.log('system','لا يمكن نسخ المصدر كما هو ('+why+') — ترميز خفيف ('+mode+') لا fixed720 الثقيل')
|
|
|
|
| 264 |
return mode
|
| 265 |
def set_probe(self,probe):
|
| 266 |
"""Injected by the manager: probe(candidate) -> {'video':..,'audio':..} or None."""
|
|
|
|
| 348 |
def command(self,destination,candidate=None):
|
| 349 |
# إعلان Enhanced-RTMP يُقاس على الترميز المدفوع فعلًا؛ ملفات الاختبار المحلية لا بروتوكول لها.
|
| 350 |
mode=self.send_mode()
|
| 351 |
+
self.last_send_mode=mode
|
| 352 |
+
if mode=='copy':
|
| 353 |
+
if self.relay_state in ('IDLE','COPY_RECOVERY','REATTACH','ALTERNATE_SOURCE','SOURCE_FAULT',''):
|
| 354 |
+
self.relay_state='COPY_STABLE'
|
| 355 |
+
elif mode=='light':
|
| 356 |
+
self.relay_state='TRANSCODE_STABLE' if self.relay_state in ('TRANSCODE_FALLBACK','TRANSCODE_STABLE') else 'TRANSCODE_FALLBACK'
|
| 357 |
+
if self.transcode_since is None:
|
| 358 |
+
self.transcode_since=time.monotonic()
|
| 359 |
network=destination.startswith(('rtmp://','rtmps://'))
|
| 360 |
enhanced=self.enhanced_advertisement(mode,candidate) if network else []
|
| 361 |
if mode=='copy':
|
| 362 |
video=['-c:v','copy']
|
|
|
|
|
|
|
| 363 |
if not self.hevc_payload(candidate):
|
| 364 |
video += ['-bsf:v','dump_extra=freq=keyframe']
|
| 365 |
+
elif mode=='light':
|
| 366 |
+
# YouTube 720p30 floor ~1500 kbps + GOP 2s. ultrafast keeps cpu-basic
|
| 367 |
+
# usable; measured fixed720/veryfast/4000k filled 2 cores at 2 publishers.
|
| 368 |
+
width,height,bitrate=720,1280,(os.getenv('YOUTUBE_LIGHT_BITRATE','1800k').strip() or '1800k')
|
| 369 |
+
try:
|
| 370 |
+
bufsize=str(int(str(bitrate).rstrip('kK'))*2)+'k'
|
| 371 |
+
except (TypeError, ValueError):
|
| 372 |
+
bufsize='3600k'
|
| 373 |
+
video=['-vf',self.canvas(width,height),'-c:v','libx264','-preset','ultrafast',
|
| 374 |
+
'-tune','zerolatency','-profile:v','main',
|
| 375 |
+
'-b:v',bitrate,'-maxrate',bitrate,'-bufsize',bufsize,
|
| 376 |
+
'-force_key_frames','expr:gte(t,n_forced*2)',
|
| 377 |
+
'-g','60','-keyint_min','30','-sc_threshold','0',
|
| 378 |
+
'-pix_fmt','yuv420p','-fps_mode','cfr']
|
| 379 |
else:
|
| 380 |
width,height,bitrate=(1080,1920,'6000k') if mode=='fixed1080' else (720,1280,'4000k')
|
| 381 |
# VIDEO_BITRATE was documented in the README but never read anywhere; it now caps the
|
|
|
|
| 658 |
sess['stall_reason'] = ''
|
| 659 |
sess['force_stop'] = False
|
| 660 |
sess['timeouts'] = 0
|
| 661 |
+
# Do NOT refresh sess['last'] here: HTTP open is not a video byte.
|
| 662 |
+
# Resetting last turned a 12s source stall into 8×reattach of a frozen picture.
|
| 663 |
sess['reattaches'] = int(sess.get('reattaches') or 0) + 1
|
| 664 |
self.current_candidate = cand
|
| 665 |
self.set(source=cand.get('label') or cand.get('brief') or '', stalled=False)
|
|
|
|
| 854 |
# ---- الحارس: لا تعتمد على poll() ولا على رمز الخروج ----
|
| 855 |
stall_limit = self._stall_seconds()
|
| 856 |
progress_limit = max(30.0, float(os.getenv('YOUTUBE_PROGRESS_STALL_SECONDS', '60')))
|
| 857 |
+
# Measured COPY_RECOVERY: origin 1920 copy after light 720 left pid alive,
|
| 858 |
+
# source bytes flowing, out_time=0 for the full 60s progress window.
|
| 859 |
+
# Zero output while the source is fresh is a failed start, not a mid-stream freeze.
|
| 860 |
+
start_limit = max(5.0, float(os.getenv('YOUTUBE_OUTPUT_START_SECONDS', '12')))
|
| 861 |
+
started_at = time.monotonic()
|
| 862 |
next_report = 0.0
|
| 863 |
while not stop.is_set():
|
| 864 |
now = time.monotonic()
|
| 865 |
+
if progress['seconds'] < 1.0 and now - started_at > start_limit:
|
| 866 |
+
src_fresh = (now - (sess['ff_last'] if response is None else sess['last'])) < max(stall_limit or 12.0, 12.0)
|
| 867 |
+
if src_fresh or response is None:
|
| 868 |
+
raise SourceStalled('FFmpeg لم يُخرج إطارًا منذ %.0f ث والمصدر يتدفق (out=0)' % (now - started_at))
|
| 869 |
# الدليل على أن المصدر لم يصمت هو بايتات تيك توك نفسها. تقدّم FFmpeg وحده لا
|
| 870 |
# يكفي: في وضع النسخ يفرّغ المخزن الداخلي لدقائق بعد صمت المصدر، فيبدو البث
|
| 871 |
# حيًا ويوتيوب يبقى على آخر ثانية — وهذا هو التعليق الذي شكا منه المستخدم.
|
| 872 |
# في HLS يستحيل القياس إلا على FFmpeg (هو من يفتح القائمة)، فنقبل تقدّمه.
|
| 873 |
_fresh = (sess['ff_last'] if response is None else sess['last'])
|
| 874 |
+
freeze_limit = max(float(stall_limit or 12.0), 12.0)
|
| 875 |
+
# Frozen picture: out_time stopped. Splicing TikTok into the same FFmpeg
|
| 876 |
+
# does not create IDRs and leaves YouTube on the last frame (measured
|
| 877 |
+
# mrjooj_7: out=123s for 60s, health still good, source_age 20→61).
|
| 878 |
+
if progress['seconds'] >= 1.0 and now - progress['last'] > freeze_limit:
|
| 879 |
+
raise SourceStalled('تجمّد خرج FFmpeg منذ %.0f ث — نعيد copy على نفس حدث يوتيوب' % (now - progress['last']))
|
| 880 |
if stall_limit and not sess.get('swapping') and now - _fresh > stall_limit:
|
| 881 |
+
if progress['seconds'] >= 1.0 and now - progress['last'] > stall_limit:
|
| 882 |
+
raise SourceStalled('المصدر صامت وخرج FFmpeg متوقف منذ %.0f ث' % (now - progress['last']))
|
| 883 |
attached=None
|
| 884 |
for _try in range(3):
|
| 885 |
attached = self._try_reattach(user, p, bridge, sess, stop, refresh,
|
youtube_manager.py
CHANGED
|
@@ -55,7 +55,7 @@ class ManagedYouTube:
|
|
| 55 |
# حتى لو كانت روابطه الموقّعة ما زالت صالحة لساعات. YOUTUBE_OBS_HOLD_SECONDS يضبط
|
| 56 |
# السقف، و0 يطفئ الاحتفاظ.
|
| 57 |
self.hold_seconds=max(0.0,float(os.getenv('YOUTUBE_OBS_HOLD_SECONDS','900')))
|
| 58 |
-
self._held={};self._last_force_at={};self.hammer={};self.retry_at={};self._offline_logged=set();self._gate_note_at={};self.auth_pause={};self.starve={};self._hold_notes={};self.last_seen_status={};self._blocked403={};self._skip_probe_once={};self._skip_hold=set();self._loop_miss={};self._ingest_bad_since={};self._good_copy={};self._spawn_at={}
|
| 59 |
# هل تقبل دالة الرصد المحقونة force=؟ (app.py يفعّلها؛ الاختبارات قد لا تفعل)
|
| 60 |
try:self._observe_force='force' in inspect.signature(self.observe).parameters
|
| 61 |
except (TypeError,ValueError):self._observe_force=False
|
|
@@ -172,8 +172,11 @@ class ManagedYouTube:
|
|
| 172 |
if w and h:
|
| 173 |
# 720x1440 vertical is 720-class (~1 Mbps), not a 1440p master.
|
| 174 |
return min(w,h)>=1440 or (w*h)>=(1600*900)
|
| 175 |
-
text=str(c.get('label') or c.get('brief') or '')
|
| 176 |
-
|
|
|
|
|
|
|
|
|
|
| 177 |
if busy>=2:
|
| 178 |
light=[c for c in out if not _too_heavy(c)]
|
| 179 |
if light:
|
|
@@ -322,13 +325,20 @@ class ManagedYouTube:
|
|
| 322 |
صلاحية/حساب = قفل طويل. سقف متزامن أو خطأ عابر = قفل قصير يُفرَّغ عندما ينتهي
|
| 323 |
حدث شقيق حتى يُفتح copy للمذيع المنتظر دون ترميز.
|
| 324 |
"""
|
|
|
|
|
|
|
| 325 |
_keys=[k for k,ev in self.journal.read()['events'].items() if ev.get('username')==user and not ev.get('closing')]
|
| 326 |
for _k in _keys:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 327 |
self.event_note(_k,'يوتيوب رفض إنشاء/تحديث الحدث (%s): تحقّق من صلاحية البث المباشر وسقف الأحداث في حسابك' % reason)
|
| 328 |
try:self.finish_event(_k)
|
| 329 |
except Exception:self.archive_event(_k,'local_cancelled_'+str(reason))
|
| 330 |
_rid=str(room_id or '')
|
| 331 |
-
hard=self._403_is_hard(reason)
|
| 332 |
limit=float(os.getenv('YOUTUBE_403_BLOCK_SECONDS') or (600 if hard else 90))
|
| 333 |
if _rid:
|
| 334 |
self._blocked403[user+':'+_rid]={'t':time.monotonic(),'reason':str(reason or ''),'hard':hard}
|
|
@@ -379,7 +389,7 @@ class ManagedYouTube:
|
|
| 379 |
with self.operations:
|
| 380 |
d=self.journal.read()
|
| 381 |
clean={k:str(values.get(k,'')).strip() for k in ('name','privacy','title','description','send_mode')}
|
| 382 |
-
if clean['send_mode'] and clean['send_mode'] not in ('auto','copy','fixed1080','fixed720'):
|
| 383 |
raise ValueError('وضع الإرسال يجب أن يكون auto أو copy أو fixed1080 أو fixed720')
|
| 384 |
if clean['privacy'] not in ('public','unlisted','private'):raise ValueError('خصوصية غير صالحة')
|
| 385 |
if 'send_mode' in values:
|
|
@@ -792,6 +802,12 @@ class ManagedYouTube:
|
|
| 792 |
# حين تبدأ الحافة بقبولنا فعلًا يُمسح العقاب وأثره — لا يُتركان حتى أول انقطاع لاحق
|
| 793 |
sender.on_stable=lambda u=ev['username']: self.mark_stream_stable(u)
|
| 794 |
sender.mode_override=self.send_mode_setting() or None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 795 |
thread=threading.Thread(target=self.sender_loop,
|
| 796 |
args=(key,ev['username'],ev['room_id'],destination,stop,sender),daemon=True)
|
| 797 |
self.senders[key]={'thread':thread,'stop':stop,'sender':sender}
|
|
@@ -925,12 +941,16 @@ class ManagedYouTube:
|
|
| 925 |
# الوجهة تُقرأ في كل دورة: لو تغيّر مفتاح الاستقبال (إعادة استخدام المصدر أو
|
| 926 |
# إعادة ربط للحدث) بقي هذا الخيط يطرق الباب القديم بمفتاح قديم إلى الأبد.
|
| 927 |
destination = self.destinations.get(user) or destination
|
|
|
|
|
|
|
|
|
|
| 928 |
try:
|
| 929 |
self.log(user,'[RELAY] @%s starting (%s)'%(user,self._source_brief(candidate)))
|
| 930 |
if sender.send(user,candidate,destination,stop, refresh=lambda: self._refresh_flv(user)):sent=True
|
| 931 |
except RelayStartError as e:
|
| 932 |
self.log(user,'[RELAY] @%s start failed; trying fallback (%s)'%(user,self.safe_error(e)[:90]))
|
| 933 |
if candidate is not None:self.mark_unstable(user,candidate,'start failed')
|
|
|
|
| 934 |
continue
|
| 935 |
except SourceStalled as e:
|
| 936 |
# Signed URLs in the hold cache are stale once bytes stop. Drop the
|
|
@@ -950,6 +970,7 @@ class ManagedYouTube:
|
|
| 950 |
stalled=True
|
| 951 |
self._skip_probe_once[user]=True
|
| 952 |
self.retry[user]=self.retry_min
|
|
|
|
| 953 |
break
|
| 954 |
except FormatChanged:
|
| 955 |
# Source resolution/layout changed (PK round started or ended, app
|
|
@@ -1006,6 +1027,7 @@ class ManagedYouTube:
|
|
| 1006 |
cand=getattr(snd,'current_candidate',None) if snd else None
|
| 1007 |
if cand and cand.get('url'):
|
| 1008 |
self._good_copy[user]=dict(cand)
|
|
|
|
| 1009 |
break
|
| 1010 |
except Exception:
|
| 1011 |
pass
|
|
@@ -1206,6 +1228,23 @@ class ManagedYouTube:
|
|
| 1206 |
preferred.insert(0, _alt)
|
| 1207 |
self.log(user, '[WATCHDOG] @%s FLV starved; trying fallback %s'%(user,str(_alt.get('media') or 'alt').upper()))
|
| 1208 |
ordered=preferred+demoted
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1209 |
self.log(user,'[SOURCE] @%s selecting source (%d candidates)'%(user,len(ordered)))
|
| 1210 |
if self._skip_probe_once.pop(user, False):
|
| 1211 |
# After a stall, HTTP probe of every rendition took ~15s of relays=0 (measured).
|
|
@@ -1711,11 +1750,14 @@ class ManagedYouTube:
|
|
| 1711 |
if life=='live' and not ev['was_live']:
|
| 1712 |
self.event_update(key,was_live=True,phase='live')
|
| 1713 |
self.log(user,'[LIVE] يوتيوب أكد أن الحدث مباشر: '+ev['broadcast_id'])
|
| 1714 |
-
# liveStarting
|
|
|
|
|
|
|
| 1715 |
self.start_sender(key)
|
| 1716 |
self.stream_health(key,user,ev)
|
| 1717 |
ev=self.journal.read()['events'].get(key) or ev
|
| 1718 |
self._ingest_copy_watchdog(key,user,ev)
|
|
|
|
| 1719 |
return
|
| 1720 |
|
| 1721 |
# life is expected to be 'ready' or 'testing' for a newly bound event
|
|
@@ -1748,15 +1790,133 @@ class ManagedYouTube:
|
|
| 1748 |
elif not ev.get('note'):
|
| 1749 |
self.event_note(key,'انتظار تأكيد استقبال يوتيوب قبل بدء الحدث')
|
| 1750 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1751 |
def _ingest_copy_watchdog(self,key,user,ev):
|
| 1752 |
"""Restart copy onto the same broadcast when YouTube ingest is dead but TikTok still flows.
|
| 1753 |
|
| 1754 |
-
FFmpeg alive is not proof of ingest. noData/bad + fresh source_age →
|
| 1755 |
-
|
| 1756 |
"""
|
| 1757 |
health=(ev.get('youtube_health') or '')
|
| 1758 |
if health in ('good','ok'):
|
| 1759 |
self._ingest_bad_since.pop(key,None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1760 |
return
|
| 1761 |
if health not in ('noData','bad','noStream'):
|
| 1762 |
return
|
|
@@ -1786,6 +1946,12 @@ class ManagedYouTube:
|
|
| 1786 |
if now-since<wait:
|
| 1787 |
return
|
| 1788 |
self._ingest_bad_since[key]=now
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1789 |
self.log(user,'[WATCHDOG] يوتيوب ingest=%s والمصدر يتدفق (age=%s ث)؛ إعادة copy على نفس الحدث %s'%(health,age,ev.get('broadcast_id') or ''))
|
| 1790 |
self.stop_sender(key)
|
| 1791 |
|
|
@@ -1816,10 +1982,32 @@ class ManagedYouTube:
|
|
| 1816 |
self.event_update(key,youtube_health=health.get('status',''),youtube_issues=text,last_issues=text)
|
| 1817 |
if text and text!=prev:
|
| 1818 |
self.log(user,'يوتيوب يشخّص البث: '+text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1819 |
def run_manager(self):
|
| 1820 |
checked=False
|
| 1821 |
last_tick=0.0
|
| 1822 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1823 |
while not self.manager_stop.is_set():
|
| 1824 |
delay=self.scan
|
| 1825 |
try:
|
|
|
|
| 55 |
# حتى لو كانت روابطه الموقّعة ما زالت صالحة لساعات. YOUTUBE_OBS_HOLD_SECONDS يضبط
|
| 56 |
# السقف، و0 يطفئ الاحتفاظ.
|
| 57 |
self.hold_seconds=max(0.0,float(os.getenv('YOUTUBE_OBS_HOLD_SECONDS','900')))
|
| 58 |
+
self._held={};self._last_force_at={};self.hammer={};self.retry_at={};self._offline_logged=set();self._gate_note_at={};self.auth_pause={};self.starve={};self._hold_notes={};self.last_seen_status={};self._blocked403={};self._skip_probe_once={};self._skip_hold=set();self._loop_miss={};self._ingest_bad_since={};self._good_copy={};self._spawn_at={};self._recover_copy=set();self._last_out={}
|
| 59 |
# هل تقبل دالة الرصد المحقونة force=؟ (app.py يفعّلها؛ الاختبارات قد لا تفعل)
|
| 60 |
try:self._observe_force='force' in inspect.signature(self.observe).parameters
|
| 61 |
except (TypeError,ValueError):self._observe_force=False
|
|
|
|
| 172 |
if w and h:
|
| 173 |
# 720x1440 vertical is 720-class (~1 Mbps), not a 1440p master.
|
| 174 |
return min(w,h)>=1440 or (w*h)>=(1600*900)
|
| 175 |
+
text=str(c.get('label') or c.get('brief') or c.get('quality_name') or '').lower()
|
| 176 |
+
# Missing width/height: "origin 1920p" must not rank as a 720 copy target.
|
| 177 |
+
if 'origin' in text or '1920' in text or '2160' in text or '4k' in text:
|
| 178 |
+
return True
|
| 179 |
+
return False
|
| 180 |
if busy>=2:
|
| 181 |
light=[c for c in out if not _too_heavy(c)]
|
| 182 |
if light:
|
|
|
|
| 325 |
صلاحية/حساب = قفل طويل. سقف متزامن أو خطأ عابر = قفل قصير يُفرَّغ عندما ينتهي
|
| 326 |
حدث شقيق حتى يُفتح copy للمذيع المنتظر دون ترميز.
|
| 327 |
"""
|
| 328 |
+
hard=self._403_is_hard(reason)
|
| 329 |
+
keep=('ready','testing','live','liveStarting')
|
| 330 |
_keys=[k for k,ev in self.journal.read()['events'].items() if ev.get('username')==user and not ev.get('closing')]
|
| 331 |
for _k in _keys:
|
| 332 |
+
ev=self.journal.read()['events'].get(_k) or {}
|
| 333 |
+
# Soft 403 (quota/rateLimit) must not tear down a bound event that already
|
| 334 |
+
# needs FFmpeg. Finish only planned/unbound shells that cannot ingest.
|
| 335 |
+
if (not hard) and ev.get('bound') and ev.get('broadcast_id') and str(ev.get('youtube_status') or '') in keep:
|
| 336 |
+
self.log(user,'[MONITOR] 403/%s على إنشاء جديد؛ نبقي الحدث القائم %s والحالة %s'%(reason, ev.get('broadcast_id'), ev.get('youtube_status')))
|
| 337 |
+
continue
|
| 338 |
self.event_note(_k,'يوتيوب رفض إنشاء/تحديث الحدث (%s): تحقّق من صلاحية البث المباشر وسقف الأحداث في حسابك' % reason)
|
| 339 |
try:self.finish_event(_k)
|
| 340 |
except Exception:self.archive_event(_k,'local_cancelled_'+str(reason))
|
| 341 |
_rid=str(room_id or '')
|
|
|
|
| 342 |
limit=float(os.getenv('YOUTUBE_403_BLOCK_SECONDS') or (600 if hard else 90))
|
| 343 |
if _rid:
|
| 344 |
self._blocked403[user+':'+_rid]={'t':time.monotonic(),'reason':str(reason or ''),'hard':hard}
|
|
|
|
| 389 |
with self.operations:
|
| 390 |
d=self.journal.read()
|
| 391 |
clean={k:str(values.get(k,'')).strip() for k in ('name','privacy','title','description','send_mode')}
|
| 392 |
+
if clean['send_mode'] and clean['send_mode'] not in ('auto','copy','light','fixed1080','fixed720'):
|
| 393 |
raise ValueError('وضع الإرسال يجب أن يكون auto أو copy أو fixed1080 أو fixed720')
|
| 394 |
if clean['privacy'] not in ('public','unlisted','private'):raise ValueError('خصوصية غير صالحة')
|
| 395 |
if 'send_mode' in values:
|
|
|
|
| 802 |
# حين تبدأ الحافة بقبولنا فعلًا يُمسح العقاب وأثره — لا يُتركان حتى أول انقطاع لاحق
|
| 803 |
sender.on_stable=lambda u=ev['username']: self.mark_stream_stable(u)
|
| 804 |
sender.mode_override=self.send_mode_setting() or None
|
| 805 |
+
if ev.get('force_light'):
|
| 806 |
+
cap=max(0,int(os.getenv('YOUTUBE_LIGHT_MAX','1')))
|
| 807 |
+
if cap>0 and self._light_count()<cap:
|
| 808 |
+
sender.sticky_mode='light'
|
| 809 |
+
else:
|
| 810 |
+
self.log(ev['username'],'[RELAY] سقف الترميز الخفيف ممتلئ عند الإقلاع؛ نُبقي copy')
|
| 811 |
thread=threading.Thread(target=self.sender_loop,
|
| 812 |
args=(key,ev['username'],ev['room_id'],destination,stop,sender),daemon=True)
|
| 813 |
self.senders[key]={'thread':thread,'stop':stop,'sender':sender}
|
|
|
|
| 941 |
# الوجهة تُقرأ في كل دورة: لو تغيّر مفتاح الاستقبال (إعادة استخدام المصدر أو
|
| 942 |
# إعادة ربط للحدث) بقي هذا الخيط يطرق الباب القديم بمفتاح قديم إلى الأبد.
|
| 943 |
destination = self.destinations.get(user) or destination
|
| 944 |
+
if getattr(sender,'sticky_mode',None)=='light' and self._light_count(except_sender=sender)>=max(0,int(os.getenv('YOUTUBE_LIGHT_MAX','1'))):
|
| 945 |
+
sender.sticky_mode=None
|
| 946 |
+
self.log(user,'[RELAY] سقف الترميز الخفيف ممتلئ؛ هذا المذيع يبقى copy')
|
| 947 |
try:
|
| 948 |
self.log(user,'[RELAY] @%s starting (%s)'%(user,self._source_brief(candidate)))
|
| 949 |
if sender.send(user,candidate,destination,stop, refresh=lambda: self._refresh_flv(user)):sent=True
|
| 950 |
except RelayStartError as e:
|
| 951 |
self.log(user,'[RELAY] @%s start failed; trying fallback (%s)'%(user,self.safe_error(e)[:90]))
|
| 952 |
if candidate is not None:self.mark_unstable(user,candidate,'start failed')
|
| 953 |
+
self._copy_recovery_miss(key,user,'start failed')
|
| 954 |
continue
|
| 955 |
except SourceStalled as e:
|
| 956 |
# Signed URLs in the hold cache are stale once bytes stop. Drop the
|
|
|
|
| 970 |
stalled=True
|
| 971 |
self._skip_probe_once[user]=True
|
| 972 |
self.retry[user]=self.retry_min
|
| 973 |
+
self._copy_recovery_miss(key,user,'stalled out=0')
|
| 974 |
break
|
| 975 |
except FormatChanged:
|
| 976 |
# Source resolution/layout changed (PK round started or ended, app
|
|
|
|
| 1027 |
cand=getattr(snd,'current_candidate',None) if snd else None
|
| 1028 |
if cand and cand.get('url'):
|
| 1029 |
self._good_copy[user]=dict(cand)
|
| 1030 |
+
self._recover_copy.discard(user)
|
| 1031 |
break
|
| 1032 |
except Exception:
|
| 1033 |
pass
|
|
|
|
| 1228 |
preferred.insert(0, _alt)
|
| 1229 |
self.log(user, '[WATCHDOG] @%s FLV starved; trying fallback %s'%(user,str(_alt.get('media') or 'alt').upper()))
|
| 1230 |
ordered=preferred+demoted
|
| 1231 |
+
if user in getattr(self,'_recover_copy',set()):
|
| 1232 |
+
def _shortside(c):
|
| 1233 |
+
try:return min(int(c.get('width') or 0), int(c.get('height') or 0))
|
| 1234 |
+
except (TypeError, ValueError):return 0
|
| 1235 |
+
def _720(c):
|
| 1236 |
+
s=_shortside(c)
|
| 1237 |
+
if 700<=s<=900:return True
|
| 1238 |
+
lab=str(c.get('label') or c.get('brief') or c.get('quality_name') or '').lower()
|
| 1239 |
+
if 'origin' in lab or '1920' in lab:return False
|
| 1240 |
+
return 'hd' in lab and 'h264' in lab
|
| 1241 |
+
ranked=[c for c in ordered if _720(c)]
|
| 1242 |
+
rest=[c for c in ordered if c not in ranked]
|
| 1243 |
+
if ranked:
|
| 1244 |
+
ordered=ranked+rest
|
| 1245 |
+
self.log(user,'[SOURCE] @%s COPY_RECOVERY: %d مرشح فئة 720 قبل origin'%(user,len(ranked)))
|
| 1246 |
+
else:
|
| 1247 |
+
self.log(user,'[SOURCE] @%s COPY_RECOVERY بلا فئة 720 — نُبقي الترتيب ونُمهل out=0 12ث'%user)
|
| 1248 |
self.log(user,'[SOURCE] @%s selecting source (%d candidates)'%(user,len(ordered)))
|
| 1249 |
if self._skip_probe_once.pop(user, False):
|
| 1250 |
# After a stall, HTTP probe of every rendition took ~15s of relays=0 (measured).
|
|
|
|
| 1750 |
if life=='live' and not ev['was_live']:
|
| 1751 |
self.event_update(key,was_live=True,phase='live')
|
| 1752 |
self.log(user,'[LIVE] يوتيوب أكد أن الحدث مباشر: '+ev['broadcast_id'])
|
| 1753 |
+
# liveStarting = transition already requested. Stopping FFmpeg here is the
|
| 1754 |
+
# spinner: YouTube waits for ingest, we refuse to send. Measured On @a3dam_111
|
| 1755 |
+
# 2026-09-28: youtube_status=liveStarting, relay_pid=None, events=1.
|
| 1756 |
self.start_sender(key)
|
| 1757 |
self.stream_health(key,user,ev)
|
| 1758 |
ev=self.journal.read()['events'].get(key) or ev
|
| 1759 |
self._ingest_copy_watchdog(key,user,ev)
|
| 1760 |
+
self._output_freeze_watchdog(key,user,ev)
|
| 1761 |
return
|
| 1762 |
|
| 1763 |
# life is expected to be 'ready' or 'testing' for a newly bound event
|
|
|
|
| 1790 |
elif not ev.get('note'):
|
| 1791 |
self.event_note(key,'انتظار تأكيد استقبال يوتيوب قبل بدء الحدث')
|
| 1792 |
|
| 1793 |
+
|
| 1794 |
+
|
| 1795 |
+
|
| 1796 |
+
def _copy_recovery_miss(self, key, user, why):
|
| 1797 |
+
if user not in getattr(self,'_recover_copy',set()):
|
| 1798 |
+
return
|
| 1799 |
+
ev=self.journal.read()['events'].get(key) or {}
|
| 1800 |
+
n=int(ev.get('copy_recovery_fail') or 0)+1
|
| 1801 |
+
self.event_update(key, copy_recovery_fail=n)
|
| 1802 |
+
self.log(user,'[RELAY] COPY_RECOVERY miss ×%d (%s)'%(n, why))
|
| 1803 |
+
if n>=2:
|
| 1804 |
+
self._recover_copy.discard(user)
|
| 1805 |
+
entry=self.senders.get(key) or {}
|
| 1806 |
+
snd=entry.get('sender')
|
| 1807 |
+
if snd and self._maybe_light(user, snd, 'copy recovery out=0 ×%d'%n, key=key):
|
| 1808 |
+
self.log(user,'[RELAY] نعود light على نفس الحدث بعد فشل copy recovery')
|
| 1809 |
+
|
| 1810 |
+
def _maybe_copy_recovery(self, key, user, ev, snd):
|
| 1811 |
+
"""Hysteresis: light → copy only after H264 + YT health good for recovery window.
|
| 1812 |
+
|
| 1813 |
+
First H264 packet is not enough (oscillation). Window default 45s = one fast
|
| 1814 |
+
YouTube health cycle (measured YOUTUBE_HEALTH_FAST_INTERVAL).
|
| 1815 |
+
"""
|
| 1816 |
+
if not snd or getattr(snd,'sticky_mode',None)!='light':
|
| 1817 |
+
if snd: snd.copy_ok_since=None
|
| 1818 |
+
return False
|
| 1819 |
+
health=(ev.get('youtube_health') or getattr(snd,'yt_health','') or '')
|
| 1820 |
+
cand=snd.current_candidate or {}
|
| 1821 |
+
blob=(str(cand.get('codec') or '')+' '+str(cand.get('label') or '')+' '+str(cand.get('brief') or '')).lower()
|
| 1822 |
+
h264=any(k in blob for k in ('h264','avc'))
|
| 1823 |
+
if health not in ('good','ok') or not h264:
|
| 1824 |
+
snd.copy_ok_since=None
|
| 1825 |
+
return False
|
| 1826 |
+
now=time.monotonic()
|
| 1827 |
+
if snd.copy_ok_since is None:
|
| 1828 |
+
snd.copy_ok_since=now
|
| 1829 |
+
snd.relay_state='TRANSCODE_STABLE'
|
| 1830 |
+
return False
|
| 1831 |
+
need=float(os.getenv('YOUTUBE_COPY_RECOVERY_SECONDS','45'))
|
| 1832 |
+
if now-snd.copy_ok_since<need:
|
| 1833 |
+
return False
|
| 1834 |
+
fails=int(ev.get('copy_recovery_fail') or 0)
|
| 1835 |
+
if fails>=2:
|
| 1836 |
+
self.log(user,'[RELAY] COPY_RECOVERY فشل مرتين هذه الجلسة؛ نُبقي light (لا oscillation)')
|
| 1837 |
+
return False
|
| 1838 |
+
snd.sticky_mode=None
|
| 1839 |
+
snd.relay_state='COPY_RECOVERY'
|
| 1840 |
+
snd.transcode_reason=''
|
| 1841 |
+
self._held.pop(user, None)
|
| 1842 |
+
self._skip_hold.add(user)
|
| 1843 |
+
getattr(self,'_recover_copy',None)
|
| 1844 |
+
if not hasattr(self,'_recover_copy'):
|
| 1845 |
+
self._recover_copy=set()
|
| 1846 |
+
self._recover_copy.add(user)
|
| 1847 |
+
self.event_update(key, force_light=False, copy_recovery=True)
|
| 1848 |
+
self.log(user,'[RELAY] H264 مستقر %.0fs وyoutube_health=%s — عودة copy (فئة 720، مصدر منعش) على نفس الحدث'%(need,health))
|
| 1849 |
+
self.stop_sender(key)
|
| 1850 |
+
return True
|
| 1851 |
+
|
| 1852 |
+
def _light_count(self, except_sender=None):
|
| 1853 |
+
n=0
|
| 1854 |
+
for e in (self.senders or {}).values():
|
| 1855 |
+
snd=e.get('sender')
|
| 1856 |
+
if not snd or snd is except_sender:
|
| 1857 |
+
continue
|
| 1858 |
+
try:
|
| 1859 |
+
if snd.sticky_mode=='light' or snd.send_mode()=='light':
|
| 1860 |
+
n+=1
|
| 1861 |
+
except Exception:
|
| 1862 |
+
pass
|
| 1863 |
+
return n
|
| 1864 |
+
|
| 1865 |
+
def _maybe_light(self, user, sender, why, key=None):
|
| 1866 |
+
cap=max(0,int(os.getenv('YOUTUBE_LIGHT_MAX','1')))
|
| 1867 |
+
if cap<=0:
|
| 1868 |
+
return False
|
| 1869 |
+
if self._light_count(except_sender=sender)>=cap:
|
| 1870 |
+
self.log(user,'[RELAY] ترميز خفيف عند السقف (%d)؛ نُبقي copy'%cap)
|
| 1871 |
+
return False
|
| 1872 |
+
sender.sticky_mode='light'
|
| 1873 |
+
sender.relay_state='TRANSCODE_FALLBACK'
|
| 1874 |
+
sender.transcode_reason=str(why or '')[:160]
|
| 1875 |
+
sender.transcode_since=time.monotonic()
|
| 1876 |
+
sender.copy_ok_since=None
|
| 1877 |
+
if key:
|
| 1878 |
+
self.event_update(key, force_light=True)
|
| 1879 |
+
self.log(user,'[RELAY] copy لم يحقق ingest مستقر (%s) — ترميز خفيف ultrafast 1800k GOP 2ث'%why)
|
| 1880 |
+
return True
|
| 1881 |
+
|
| 1882 |
+
def _output_freeze_watchdog(self, key, user, ev):
|
| 1883 |
+
"""YouTube health=good is not a moving picture. Frozen out_time = frozen frame.
|
| 1884 |
+
|
| 1885 |
+
Measured: mrjooj_7 copy out=123s for ~60s, source_age 61s, youtube_health=good.
|
| 1886 |
+
"""
|
| 1887 |
+
entry=self.senders.get(key) or {}
|
| 1888 |
+
snd=entry.get('sender')
|
| 1889 |
+
try:
|
| 1890 |
+
snap=snd.snapshot() if snd else {}
|
| 1891 |
+
out=float((snap or {}).get('output_seconds') or 0)
|
| 1892 |
+
alive=bool(snd and snd.alive())
|
| 1893 |
+
except Exception:
|
| 1894 |
+
return
|
| 1895 |
+
now=time.monotonic()
|
| 1896 |
+
prev=self._last_out.get(key)
|
| 1897 |
+
if out>=1.0 and prev and out<=prev[0]+0.15:
|
| 1898 |
+
if alive and now-prev[1]>=12.0 and self._yt_public(ev):
|
| 1899 |
+
self.log(user,'[WATCHDOG] out_time مجمّد %.0fs (%.1fث) والصحة يوتيوب=%s — إعادة copy نفس الحدث'%(now-prev[1], out, ev.get('youtube_health') or ''))
|
| 1900 |
+
self._last_out.pop(key, None)
|
| 1901 |
+
self.stop_sender(key)
|
| 1902 |
+
return
|
| 1903 |
+
return
|
| 1904 |
+
self._last_out[key]=(out, now)
|
| 1905 |
+
|
| 1906 |
def _ingest_copy_watchdog(self,key,user,ev):
|
| 1907 |
"""Restart copy onto the same broadcast when YouTube ingest is dead but TikTok still flows.
|
| 1908 |
|
| 1909 |
+
FFmpeg alive is not proof of ingest. noData/bad + fresh source_age → restart copy;
|
| 1910 |
+
after ≥2 copy restarts escalate to light (capped). same broadcast_id.
|
| 1911 |
"""
|
| 1912 |
health=(ev.get('youtube_health') or '')
|
| 1913 |
if health in ('good','ok'):
|
| 1914 |
self._ingest_bad_since.pop(key,None)
|
| 1915 |
+
entry=self.senders.get(key) or {}
|
| 1916 |
+
snd=entry.get('sender')
|
| 1917 |
+
if snd:
|
| 1918 |
+
snd.yt_health=health
|
| 1919 |
+
self._maybe_copy_recovery(key,user,ev,snd)
|
| 1920 |
return
|
| 1921 |
if health not in ('noData','bad','noStream'):
|
| 1922 |
return
|
|
|
|
| 1946 |
if now-since<wait:
|
| 1947 |
return
|
| 1948 |
self._ingest_bad_since[key]=now
|
| 1949 |
+
retries=int(ev.get('copy_retries') or 0)+1
|
| 1950 |
+
self.event_update(key, copy_retries=retries)
|
| 1951 |
+
if retries>=2 and snd:
|
| 1952 |
+
if self._maybe_light(user, snd, 'noData/bad ×%d'%retries, key=key):
|
| 1953 |
+
self.stop_sender(key)
|
| 1954 |
+
return
|
| 1955 |
self.log(user,'[WATCHDOG] يوتيوب ingest=%s والمصدر يتدفق (age=%s ث)؛ إعادة copy على نفس الحدث %s'%(health,age,ev.get('broadcast_id') or ''))
|
| 1956 |
self.stop_sender(key)
|
| 1957 |
|
|
|
|
| 1982 |
self.event_update(key,youtube_health=health.get('status',''),youtube_issues=text,last_issues=text)
|
| 1983 |
if text and text!=prev:
|
| 1984 |
self.log(user,'يوتيوب يشخّص البث: '+text)
|
| 1985 |
+
# GOP too long / starved ingest → YouTube loading. bitrateLow alone is not enough
|
| 1986 |
+
# (measured: 3× H264 copy ~1 Mbps stayed good/ok at ~8% CPU).
|
| 1987 |
+
hard=any(k in (text or '') for k in ('gopSizeLong','openGop','videoIngestionStarved'))
|
| 1988 |
+
entry=self.senders.get(key) or {}
|
| 1989 |
+
snd=entry.get('sender')
|
| 1990 |
+
if snd:
|
| 1991 |
+
snd.yt_health=health.get('status','')
|
| 1992 |
+
if hard:
|
| 1993 |
+
if snd and self._maybe_light(user, snd, text[:80], key=key):
|
| 1994 |
+
self.stop_sender(key)
|
| 1995 |
+
return
|
| 1996 |
+
if snd:
|
| 1997 |
+
self._maybe_copy_recovery(key,user,self.journal.read()['events'].get(key) or ev,snd)
|
| 1998 |
def run_manager(self):
|
| 1999 |
checked=False
|
| 2000 |
last_tick=0.0
|
| 2001 |
try:
|
| 2002 |
+
# Journal force_light survives Space rebuild and started origin 1920 as light
|
| 2003 |
+
# at 90–100% of 2 cores (On 2026-09-28). Copy-first on boot; light only from
|
| 2004 |
+
# a live ingest fault in this process.
|
| 2005 |
+
try:
|
| 2006 |
+
for _k,_ev in list(self.journal.read()['events'].items()):
|
| 2007 |
+
if _ev.get('force_light') or _ev.get('copy_recovery_fail'):
|
| 2008 |
+
self.event_update(_k, force_light=False, copy_recovery_fail=0)
|
| 2009 |
+
except Exception:
|
| 2010 |
+
pass
|
| 2011 |
while not self.manager_stop.is_set():
|
| 2012 |
delay=self.scan
|
| 2013 |
try:
|