Revert On to a5b04d24b028 (user: a5b04d2) — copy H264 even if flap sticky, FLAP_LIMIT=4
Browse files- Dockerfile +4 -7
- app.py +10 -29
- index.html +2 -3
- ingest.py +11 -31
- relay.py +15 -108
- youtube_manager.py +60 -510
Dockerfile
CHANGED
|
@@ -13,27 +13,24 @@ ENV TZ=Asia/Baghdad \
|
|
| 13 |
CHECK_INTERVAL=20 \
|
| 14 |
SOURCE_STALL_TIMEOUT=12 \
|
| 15 |
YOUTUBE_SOURCE_STALL_SECONDS=12 \
|
| 16 |
-
MAX_CONCURRENT_RELAYS=
|
| 17 |
-
YOUTUBE_MAX_EVENTS=
|
| 18 |
YOUTUBE_TITLE_TEMPLATE="{broadcaster} | {date} {time}" \
|
| 19 |
YOUTUBE_SEND_MODE=auto \
|
| 20 |
YOUTUBE_AUTO_TRANSCODE=1 \
|
| 21 |
-
YOUTUBE_LIGHT_MAX=1 \
|
| 22 |
-
YOUTUBE_LIGHT_BITRATE=1800k \
|
| 23 |
-
YOUTUBE_COPY_RECOVERY_SECONDS=45 \
|
| 24 |
YOUTUBE_AUDIO_PROFILE=aac_low \
|
| 25 |
YOUTUBE_AUDIO_BITRATE=128k \
|
| 26 |
YOUTUBE_AUDIO_CHANNELS=2 \
|
| 27 |
YOUTUBE_GOP=60 \
|
| 28 |
YOUTUBE_INPUT_RECOVERY=1 \
|
| 29 |
YOUTUBE_PREFER_H264=1 \
|
| 30 |
-
YOUTUBE_CODEC_FALLBACK_MODE=
|
| 31 |
YOUTUBE_RETRY_MIN=1 \
|
| 32 |
YOUTUBE_RETRY_MAX=60 \
|
| 33 |
YOUTUBE_SOURCE_LIMIT=6 \
|
| 34 |
YOUTUBE_FORMAT_GUARD=on \
|
| 35 |
YOUTUBE_FLAP_LIMIT=4 \
|
| 36 |
-
YOUTUBE_FLAP_FALLBACK_MODE=
|
| 37 |
YOUTUBE_X264_PRESET=veryfast \
|
| 38 |
YOUTUBE_CRF=22 \
|
| 39 |
YOUTUBE_FIFO_TIMESHIFT=0 \
|
|
|
|
| 13 |
CHECK_INTERVAL=20 \
|
| 14 |
SOURCE_STALL_TIMEOUT=12 \
|
| 15 |
YOUTUBE_SOURCE_STALL_SECONDS=12 \
|
| 16 |
+
MAX_CONCURRENT_RELAYS=2 \
|
| 17 |
+
YOUTUBE_MAX_EVENTS=2 \
|
| 18 |
YOUTUBE_TITLE_TEMPLATE="{broadcaster} | {date} {time}" \
|
| 19 |
YOUTUBE_SEND_MODE=auto \
|
| 20 |
YOUTUBE_AUTO_TRANSCODE=1 \
|
|
|
|
|
|
|
|
|
|
| 21 |
YOUTUBE_AUDIO_PROFILE=aac_low \
|
| 22 |
YOUTUBE_AUDIO_BITRATE=128k \
|
| 23 |
YOUTUBE_AUDIO_CHANNELS=2 \
|
| 24 |
YOUTUBE_GOP=60 \
|
| 25 |
YOUTUBE_INPUT_RECOVERY=1 \
|
| 26 |
YOUTUBE_PREFER_H264=1 \
|
| 27 |
+
YOUTUBE_CODEC_FALLBACK_MODE=fixed720 \
|
| 28 |
YOUTUBE_RETRY_MIN=1 \
|
| 29 |
YOUTUBE_RETRY_MAX=60 \
|
| 30 |
YOUTUBE_SOURCE_LIMIT=6 \
|
| 31 |
YOUTUBE_FORMAT_GUARD=on \
|
| 32 |
YOUTUBE_FLAP_LIMIT=4 \
|
| 33 |
+
YOUTUBE_FLAP_FALLBACK_MODE=fixed720 \
|
| 34 |
YOUTUBE_X264_PRESET=veryfast \
|
| 35 |
YOUTUBE_CRF=22 \
|
| 36 |
YOUTUBE_FIFO_TIMESHIFT=0 \
|
app.py
CHANGED
|
@@ -537,7 +537,7 @@ def _maybe_start_direct(user, obs):
|
|
| 537 |
return
|
| 538 |
if hub.at_capacity(except_user=user):
|
| 539 |
update(user, state='BACKOFF', status='بلغنا حد MAX_CONCURRENT_RELAYS')
|
| 540 |
-
log(user,'[RELAY] @%s waiting: at MAX_CONCURRENT_RELAYS=%
|
| 541 |
return
|
| 542 |
stop=threading.Event()
|
| 543 |
title=format_session_title(obs.get('nickname') or user)[0]
|
|
@@ -596,8 +596,6 @@ def _relay_snapshot(user):
|
|
| 596 |
'relay_dest':slot.get('dest') if slot else '',
|
| 597 |
'restarts':hub.restart_count(user),
|
| 598 |
'send_mode':snap.get('send_mode') or '',
|
| 599 |
-
'relay_state':snap.get('relay_state') or '',
|
| 600 |
-
'transcode_reason':snap.get('transcode_reason') or '',
|
| 601 |
}
|
| 602 |
|
| 603 |
def _cgroup_usage_ns():
|
|
@@ -735,9 +733,7 @@ def _check_account(user):
|
|
| 735 |
log(user,'[MONITOR] @%s session/room changed; source will be refreshed'%user)
|
| 736 |
_set_state(user,'LIVE','LIVE',**extra)
|
| 737 |
_maybe_start_direct(user, obs)
|
| 738 |
-
|
| 739 |
-
# block nudge forever while /health showed LIVE and rel=0 (On 4 LIVE, 0 senders).
|
| 740 |
-
if _oauth_automation_on() and (obs.get('sources') or []):
|
| 741 |
try:
|
| 742 |
youtube.nudge_live(user, obs)
|
| 743 |
except Exception:
|
|
@@ -790,15 +786,9 @@ def _pick_last_error(tasks_map):
|
|
| 790 |
return err, u
|
| 791 |
return last_error, last_error_user
|
| 792 |
|
| 793 |
-
_health_cache={'t':0.0,'v':None}
|
| 794 |
-
|
| 795 |
@app.get('/health')
|
| 796 |
@app.get('/healthz')
|
| 797 |
def health_public():
|
| 798 |
-
now=time.monotonic()
|
| 799 |
-
cached=_health_cache.get('v')
|
| 800 |
-
if cached is not None and now-float(_health_cache.get('t') or 0)<2.0:
|
| 801 |
-
return cached
|
| 802 |
with lock:
|
| 803 |
n=len(tasks);enabled=sum(1 for t in tasks.values() if t.get('enabled'))
|
| 804 |
last_error, last_error_user=_pick_last_error(tasks)
|
|
@@ -824,31 +814,29 @@ def health_public():
|
|
| 824 |
'last_error':tsk.get('last_error') or '','source_age':rs.get('source_age'),
|
| 825 |
'output_seconds':rs.get('output_seconds') or 0,
|
| 826 |
'send_mode':rs.get('send_mode') or '',
|
| 827 |
-
'relay_state':rs.get('relay_state') or '',
|
| 828 |
-
'transcode_reason':rs.get('transcode_reason') or '',
|
| 829 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 830 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 831 |
'youtube_status':str(ye.get('youtube_status') or '')})
|
| 832 |
res=resource_snapshot()
|
| 833 |
mgr=bool(youtube.manager_thread and youtube.manager_thread.is_alive())
|
| 834 |
-
y_events=0; y_hist=0; y_senders=0; y_status=''; y_skip=''
|
| 835 |
try:
|
| 836 |
jd=youtube.journal.read()
|
| 837 |
y_events=len(jd.get('events') or {})
|
| 838 |
y_hist=len(jd.get('history') or [])
|
| 839 |
-
y_evu=','.join('%s:%s:%s:id=%s:pend=%s:%s'%(
|
| 840 |
-
e.get('username') or '?', e.get('phase') or '', e.get('youtube_status') or '',
|
| 841 |
-
'1' if e.get('broadcast_id') else '0', '1' if e.get('create_pending') else '0',
|
| 842 |
-
str(e.get('note') or '')[:40])
|
| 843 |
-
for e in (jd.get('events') or {}).values())[:320]
|
| 844 |
y_status=str(getattr(youtube,'status_text','') or '')[:160]
|
| 845 |
y_senders=sum(1 for e in (youtube.senders or {}).values() if e.get('thread') and e['thread'].is_alive())
|
| 846 |
skip=getattr(youtube,'_skip_last',None) or {}
|
| 847 |
if skip:
|
| 848 |
-
y_skip='; '.join('%s: %s'%(u,str(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 849 |
except Exception:
|
| 850 |
pass
|
| 851 |
-
|
| 852 |
'accounts':n,'enabled':enabled,'active_relays':active,
|
| 853 |
'max_concurrent_relays':max_concurrent_relays(),
|
| 854 |
'source_stall_timeout':SOURCE_STALL_TIMEOUT,
|
|
@@ -863,10 +851,7 @@ def health_public():
|
|
| 863 |
'youtube_manager':'running' if mgr else 'stopped',
|
| 864 |
'youtube_status':y_status,'youtube_events':y_events,'youtube_history':y_hist,
|
| 865 |
'youtube_senders':y_senders,'youtube_skip':y_skip,
|
| 866 |
-
'youtube_event_users':y_evu,
|
| 867 |
'broadcasters':per}
|
| 868 |
-
_health_cache['t']=time.monotonic(); _health_cache['v']=payload
|
| 869 |
-
return payload
|
| 870 |
|
| 871 |
@app.get('/api/health')
|
| 872 |
def health_detail():
|
|
@@ -941,10 +926,6 @@ def state():
|
|
| 941 |
'last_data_time':rs.get('last_data_time'),'relay_alive':rs.get('relay_alive'),
|
| 942 |
'session_title':rs.get('session_title') or '','restarts':rs.get('restarts') or 0,
|
| 943 |
'source_age':rs.get('source_age'),
|
| 944 |
-
'send_mode':rs.get('send_mode') or '',
|
| 945 |
-
'relay_state':rs.get('relay_state') or '',
|
| 946 |
-
'transcode_reason':rs.get('transcode_reason') or '',
|
| 947 |
-
'output_seconds':rs.get('output_seconds') or 0,
|
| 948 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 949 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 950 |
'youtube_status':str(ye.get('youtube_status') or ('relay' if rs.get('relay_alive') else (t.get('state') or '')))})
|
|
|
|
| 537 |
return
|
| 538 |
if hub.at_capacity(except_user=user):
|
| 539 |
update(user, state='BACKOFF', status='بلغنا حد MAX_CONCURRENT_RELAYS')
|
| 540 |
+
log(user,'[RELAY] @%s waiting: at MAX_CONCURRENT_RELAYS=%d'%(user, max_concurrent_relays()))
|
| 541 |
return
|
| 542 |
stop=threading.Event()
|
| 543 |
title=format_session_title(obs.get('nickname') or user)[0]
|
|
|
|
| 596 |
'relay_dest':slot.get('dest') if slot else '',
|
| 597 |
'restarts':hub.restart_count(user),
|
| 598 |
'send_mode':snap.get('send_mode') or '',
|
|
|
|
|
|
|
| 599 |
}
|
| 600 |
|
| 601 |
def _cgroup_usage_ns():
|
|
|
|
| 733 |
log(user,'[MONITOR] @%s session/room changed; source will be refreshed'%user)
|
| 734 |
_set_state(user,'LIVE','LIVE',**extra)
|
| 735 |
_maybe_start_direct(user, obs)
|
| 736 |
+
if _oauth_automation_on() and not hub.busy(user) and (obs.get('sources') or []):
|
|
|
|
|
|
|
| 737 |
try:
|
| 738 |
youtube.nudge_live(user, obs)
|
| 739 |
except Exception:
|
|
|
|
| 786 |
return err, u
|
| 787 |
return last_error, last_error_user
|
| 788 |
|
|
|
|
|
|
|
| 789 |
@app.get('/health')
|
| 790 |
@app.get('/healthz')
|
| 791 |
def health_public():
|
|
|
|
|
|
|
|
|
|
|
|
|
| 792 |
with lock:
|
| 793 |
n=len(tasks);enabled=sum(1 for t in tasks.values() if t.get('enabled'))
|
| 794 |
last_error, last_error_user=_pick_last_error(tasks)
|
|
|
|
| 814 |
'last_error':tsk.get('last_error') or '','source_age':rs.get('source_age'),
|
| 815 |
'output_seconds':rs.get('output_seconds') or 0,
|
| 816 |
'send_mode':rs.get('send_mode') or '',
|
|
|
|
|
|
|
| 817 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 818 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 819 |
'youtube_status':str(ye.get('youtube_status') or '')})
|
| 820 |
res=resource_snapshot()
|
| 821 |
mgr=bool(youtube.manager_thread and youtube.manager_thread.is_alive())
|
| 822 |
+
y_events=0; y_hist=0; y_senders=0; y_status=''; y_skip=''
|
| 823 |
try:
|
| 824 |
jd=youtube.journal.read()
|
| 825 |
y_events=len(jd.get('events') or {})
|
| 826 |
y_hist=len(jd.get('history') or [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 827 |
y_status=str(getattr(youtube,'status_text','') or '')[:160]
|
| 828 |
y_senders=sum(1 for e in (youtube.senders or {}).values() if e.get('thread') and e['thread'].is_alive())
|
| 829 |
skip=getattr(youtube,'_skip_last',None) or {}
|
| 830 |
if skip:
|
| 831 |
+
y_skip='; '.join('%s: %s'%(u,str(t)[:80]) for u,t in list(skip.items())[-3:])[:200]
|
| 832 |
+
try:
|
| 833 |
+
acc=list(youtube.accounts() or [])
|
| 834 |
+
y_skip=(y_skip+(' | accounts='+','.join(acc)[:80])).strip(' |')
|
| 835 |
+
except Exception:
|
| 836 |
+
pass
|
| 837 |
except Exception:
|
| 838 |
pass
|
| 839 |
+
return {'application':'running','monitor':'running' if _monitor_alive.is_set() else 'stopped',
|
| 840 |
'accounts':n,'enabled':enabled,'active_relays':active,
|
| 841 |
'max_concurrent_relays':max_concurrent_relays(),
|
| 842 |
'source_stall_timeout':SOURCE_STALL_TIMEOUT,
|
|
|
|
| 851 |
'youtube_manager':'running' if mgr else 'stopped',
|
| 852 |
'youtube_status':y_status,'youtube_events':y_events,'youtube_history':y_hist,
|
| 853 |
'youtube_senders':y_senders,'youtube_skip':y_skip,
|
|
|
|
| 854 |
'broadcasters':per}
|
|
|
|
|
|
|
| 855 |
|
| 856 |
@app.get('/api/health')
|
| 857 |
def health_detail():
|
|
|
|
| 926 |
'last_data_time':rs.get('last_data_time'),'relay_alive':rs.get('relay_alive'),
|
| 927 |
'session_title':rs.get('session_title') or '','restarts':rs.get('restarts') or 0,
|
| 928 |
'source_age':rs.get('source_age'),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 929 |
'youtube_health':str(ye.get('youtube_health') or ''),
|
| 930 |
'youtube_issues':str(ye.get('youtube_issues') or '')[:120],
|
| 931 |
'youtube_status':str(ye.get('youtube_status') or ('relay' if rs.get('relay_alive') else (t.get('state') or '')))})
|
index.html
CHANGED
|
@@ -144,9 +144,8 @@ async function refresh(){
|
|
| 144 |
${x.nickname?' · '+esc(x.nickname):''}</p>
|
| 145 |
<p>LIVE منذ: ${live?esc(fmtDur(x.live_for||0)):'—'}</p>
|
| 146 |
<p>المصدر: ${esc(x.source||'—')}</p>
|
| 147 |
-
<p>MODE: ${esc(x.send_mode||x.relay_state||'—')}${x.transcode_reason?' · '+esc(x.transcode_reason):''}</p>
|
| 148 |
<p>Codec: ${esc(x.codec||'—')} · ${esc(x.quality||'')} · room ${esc(x.room_id||'—')}</p>
|
| 149 |
-
<p>PID: ${x.relay_pid||'—'} · بايت: ${esc(x.bytes_received||0)} · إعادة: ${esc(x.restarts||0)}
|
| 150 |
<p>آخر بايت: ${x.source_age==null||x.source_age===''?'—':(Number(x.source_age).toFixed(1)+'ث')}</p>
|
| 151 |
<p>YouTube: ${esc(x.youtube_status||'—')} · health ${esc(x.youtube_health||'—')}${x.relay_alive?' · Relay':''}</p>
|
| 152 |
<small>${esc(x.status||'')}${x.last_error?' · '+esc(x.last_error):''}</small>
|
|
@@ -182,7 +181,7 @@ async function refresh(){
|
|
| 182 |
const lastErr=tasks.map(t=>t.last_error).filter(Boolean)[0] || (s0.logs||[]).filter(l=>/error|فشل|stall|خطأ/i.test(l.message||'')).slice(-1)[0]?.message || 'لا يوجد';
|
| 183 |
$('mon').textContent=s0.monitor_running?'يعمل':'متوقف';
|
| 184 |
$('nacc').textContent=String(tasks.length);
|
| 185 |
-
|
| 186 |
$('ncpu').textContent=(s0.load1!=null?('load '+s0.load1+' / '+ (s0.cpus||'?')+' CPU'):'—');
|
| 187 |
$('nram').textContent=(s0.mem_avail_mb!=null?(s0.mem_avail_mb+' MB متاح / '+s0.mem_total_mb+' MB'):'—');
|
| 188 |
$('lasterr').textContent=lastErr;
|
|
|
|
| 144 |
${x.nickname?' · '+esc(x.nickname):''}</p>
|
| 145 |
<p>LIVE منذ: ${live?esc(fmtDur(x.live_for||0)):'—'}</p>
|
| 146 |
<p>المصدر: ${esc(x.source||'—')}</p>
|
|
|
|
| 147 |
<p>Codec: ${esc(x.codec||'—')} · ${esc(x.quality||'')} · room ${esc(x.room_id||'—')}</p>
|
| 148 |
+
<p>PID: ${x.relay_pid||'—'} · بايت: ${esc(x.bytes_received||0)} · إعادة: ${esc(x.restarts||0)}</p>
|
| 149 |
<p>آخر بايت: ${x.source_age==null||x.source_age===''?'—':(Number(x.source_age).toFixed(1)+'ث')}</p>
|
| 150 |
<p>YouTube: ${esc(x.youtube_status||'—')} · health ${esc(x.youtube_health||'—')}${x.relay_alive?' · Relay':''}</p>
|
| 151 |
<small>${esc(x.status||'')}${x.last_error?' · '+esc(x.last_error):''}</small>
|
|
|
|
| 181 |
const lastErr=tasks.map(t=>t.last_error).filter(Boolean)[0] || (s0.logs||[]).filter(l=>/error|فشل|stall|خطأ/i.test(l.message||'')).slice(-1)[0]?.message || 'لا يوجد';
|
| 182 |
$('mon').textContent=s0.monitor_running?'يعمل':'متوقف';
|
| 183 |
$('nacc').textContent=String(tasks.length);
|
| 184 |
+
$('nrel').textContent=String(s0.active_relays||evs.length||0)+' / '+(s0.max_concurrent_relays||'?');
|
| 185 |
$('ncpu').textContent=(s0.load1!=null?('load '+s0.load1+' / '+ (s0.cpus||'?')+' CPU'):'—');
|
| 186 |
$('nram').textContent=(s0.mem_avail_mb!=null?(s0.mem_avail_mb+' MB متاح / '+s0.mem_total_mb+' MB'):'—');
|
| 187 |
$('lasterr').textContent=lastErr;
|
ingest.py
CHANGED
|
@@ -28,36 +28,19 @@ def rtmp_configured():
|
|
| 28 |
return bool(env_rtmp_destination())
|
| 29 |
|
| 30 |
|
| 31 |
-
def _open_cap(raw, default='0'):
|
| 32 |
-
"""0 / open / unlimited = no global cap. Hub still 0-or-1 per broadcaster."""
|
| 33 |
-
s = str(raw if raw is not None else default).strip().lower()
|
| 34 |
-
if s in ('', '0', 'unlimited', 'open', 'none', 'off', 'no', 'false'):
|
| 35 |
-
return 0
|
| 36 |
-
try:
|
| 37 |
-
n = int(s)
|
| 38 |
-
except (TypeError, ValueError):
|
| 39 |
-
return 0
|
| 40 |
-
return n if n > 0 else 0
|
| 41 |
-
|
| 42 |
-
|
| 43 |
def max_concurrent_relays():
|
| 44 |
-
"""
|
| 45 |
|
| 46 |
-
|
| 47 |
-
|
|
|
|
| 48 |
"""
|
| 49 |
-
raw = os.getenv('MAX_CONCURRENT_RELAYS')
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def youtube_max_events():
|
| 56 |
-
"""Simultaneous YouTube events. 0 = no extra cap beyond one event per user."""
|
| 57 |
-
raw = os.getenv('YOUTUBE_MAX_EVENTS')
|
| 58 |
-
if raw is None or str(raw).strip() == '':
|
| 59 |
-
return max_concurrent_relays()
|
| 60 |
-
return _open_cap(raw, '0')
|
| 61 |
|
| 62 |
|
| 63 |
def format_session_title(broadcaster, when=None):
|
|
@@ -162,12 +145,9 @@ class RelayHub:
|
|
| 162 |
|
| 163 |
def at_capacity(self, except_user=None):
|
| 164 |
with self.lock:
|
| 165 |
-
cap = max_concurrent_relays()
|
| 166 |
-
if cap <= 0:
|
| 167 |
-
return False
|
| 168 |
n = sum(1 for u, v in self.slots.items()
|
| 169 |
if u != except_user and self._ffmpeg_alive(v))
|
| 170 |
-
return n >=
|
| 171 |
|
| 172 |
def env_key_busy(self, except_user=None):
|
| 173 |
"""A single env stream key can only accept one publisher (any owner)."""
|
|
|
|
| 28 |
return bool(env_rtmp_destination())
|
| 29 |
|
| 30 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
def max_concurrent_relays():
|
| 32 |
+
"""Cap on simultaneous FFmpeg publishers. Not a quality guarantee.
|
| 33 |
|
| 34 |
+
Default 2: cpu-basic has carried 1 comfortably and 2 at high quota; 3+ is
|
| 35 |
+
not a pass. Operators may raise MAX_CONCURRENT_RELAYS; health reports the
|
| 36 |
+
value actually in force. Ceiling 8 is only a fork-bomb guard.
|
| 37 |
"""
|
| 38 |
+
raw = os.getenv('MAX_CONCURRENT_RELAYS') or os.getenv('YOUTUBE_MAX_EVENTS') or '2'
|
| 39 |
+
try:
|
| 40 |
+
n = int(str(raw).strip())
|
| 41 |
+
except (TypeError, ValueError):
|
| 42 |
+
n = 2
|
| 43 |
+
return max(1, min(n, 8))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
|
| 46 |
def format_session_title(broadcaster, when=None):
|
|
|
|
| 145 |
|
| 146 |
def at_capacity(self, except_user=None):
|
| 147 |
with self.lock:
|
|
|
|
|
|
|
|
|
|
| 148 |
n = sum(1 for u, v in self.slots.items()
|
| 149 |
if u != except_user and self._ffmpeg_alive(v))
|
| 150 |
+
return n >= max_concurrent_relays()
|
| 151 |
|
| 152 |
def env_key_busy(self, except_user=None):
|
| 153 |
"""A single env stream key can only accept one publisher (any owner)."""
|
relay.py
CHANGED
|
@@ -100,13 +100,6 @@ class Sender:
|
|
| 100 |
# event. Without this the flap window expires after 10 quiet minutes and the
|
| 101 |
# sender silently drops back to copy mode, restarting the whole failure loop.
|
| 102 |
self.sticky_mode=None
|
| 103 |
-
self.last_send_mode=''
|
| 104 |
-
self.relay_state='IDLE'
|
| 105 |
-
self.transcode_reason=''
|
| 106 |
-
self.transcode_since=None
|
| 107 |
-
self.copy_ok_since=None
|
| 108 |
-
self.yt_health=''
|
| 109 |
-
self.session_out_codec='h264'
|
| 110 |
# Candidate handed to send_mode() so the codec rule can judge the real source.
|
| 111 |
self.current_candidate=None
|
| 112 |
# Journal setting wins over the environment so the mode can be changed from the UI
|
|
@@ -124,20 +117,9 @@ class Sender:
|
|
| 124 |
stamp=d.get('last_source_at')
|
| 125 |
if stamp:d['source_age']=round(max(0.0,time.monotonic()-stamp),1)
|
| 126 |
try:
|
| 127 |
-
d['send_mode']=self.
|
| 128 |
except Exception:
|
| 129 |
d['send_mode']=''
|
| 130 |
-
d['relay_state']=getattr(self,'relay_state','') or ''
|
| 131 |
-
d['transcode_reason']=getattr(self,'transcode_reason','') or ''
|
| 132 |
-
# Hide leftover TLS/EOF from a previous handshake while this FFmpeg is actually sending.
|
| 133 |
-
try:
|
| 134 |
-
alive=self.alive()
|
| 135 |
-
except Exception:
|
| 136 |
-
alive=False
|
| 137 |
-
if alive and float(d.get('output_seconds') or 0)>=15 and float(d.get('source_age') or 99)<3:
|
| 138 |
-
d['ffmpeg_errors']=[]
|
| 139 |
-
d['ffmpeg_diagnostics']=[]
|
| 140 |
-
d['source_gap']=0
|
| 141 |
return d
|
| 142 |
def wait_until(self,condition,stop,poll=1.0,timeout=None):
|
| 143 |
"""Block while `condition()` holds; returns as soon as it stops or a stop is requested."""
|
|
@@ -176,16 +158,12 @@ class Sender:
|
|
| 176 |
fixed720 -> constant 720x1280 canvas, re-encodes (lighter on CPU)
|
| 177 |
"""
|
| 178 |
mode=(self.mode_override or os.getenv('YOUTUBE_SEND_MODE','auto')).strip().lower()
|
| 179 |
-
if mode not in ('auto','copy','
|
| 180 |
-
raise ValueError('وضع الإرسال يجب أن يكون auto أو copy أو
|
| 181 |
# Sticky transcode is for HEVC / layout flap only. Measured 2026-09-27 on cpu-basic:
|
| 182 |
# two fixed720 publishers used 1.67–1.85 of 2.0 quota (peak 92.6%) with YouTube
|
| 183 |
# health still good — no headroom for a third. If this candidate is H264, copy.
|
| 184 |
if self.sticky_mode:
|
| 185 |
-
# light stays until the manager hysteresis (H264 + YT health good for
|
| 186 |
-
# YOUTUBE_COPY_RECOVERY_SECONDS). Do not drop on the first H264 packet.
|
| 187 |
-
if self.sticky_mode=='light':
|
| 188 |
-
return 'light'
|
| 189 |
cand=self.current_candidate or {}
|
| 190 |
blob=(str(cand.get('codec') or '')+' '+str(cand.get('label') or '')+' '+str(cand.get('brief') or '')).lower()
|
| 191 |
if any(k in blob for k in ('h264','avc')):
|
|
@@ -208,19 +186,14 @@ class Sender:
|
|
| 208 |
now=time.monotonic()
|
| 209 |
while flap and now-flap[0]>600:flap.popleft()
|
| 210 |
if len(flap)>=int(os.getenv('YOUTUBE_FLAP_LIMIT','4')):
|
| 211 |
-
|
| 212 |
-
if
|
| 213 |
-
|
| 214 |
-
mode=fb
|
| 215 |
return mode
|
| 216 |
def note_format_change(self):
|
| 217 |
self.flap.append(time.monotonic())
|
| 218 |
if len(self.flap)>=int(os.getenv('YOUTUBE_FLAP_LIMIT','4')):
|
| 219 |
-
|
| 220 |
-
if fb in ('light','fixed1080','fixed720'):
|
| 221 |
-
self.log('system','تغيّرت أبعاد المصدر عدة مرات؛ copy لم يكفِ — ترميز خفيف (%s) بمفتاح IDR كل 2ث'%fb)
|
| 222 |
-
else:
|
| 223 |
-
self.log('system','تغيّرت أبعاد المصدر عدة مرات؛ نُبقي copy ونعيد توصيل تيك توك على نفس جلسة يوتيوب')
|
| 224 |
@staticmethod
|
| 225 |
def copy_is_safe(declared,probed):
|
| 226 |
"""Stream copy is allowed only on positive proof of H.264, never on absence of evidence.
|
|
@@ -256,11 +229,12 @@ class Sender:
|
|
| 256 |
label=str(cand.get('label') or cand.get('brief') or '')
|
| 257 |
if any(k in str(declared).lower() for k in ('h264','avc')) or any(k in label.lower() for k in ('h264','avc')):
|
| 258 |
return None
|
| 259 |
-
mode=os.getenv('YOUTUBE_CODEC_FALLBACK_MODE','
|
| 260 |
-
if mode not in ('
|
| 261 |
self.sticky_mode=mode
|
| 262 |
why=('ffprobe: '+str(probed)) if probed else (('مُعلَن: '+declared) if declared else 'ترميز مجهول والفحص لم يأتِ بنتيجة')
|
| 263 |
-
self.log('system','لا يمكن نسخ المصدر كما هو ('+why+') — ت
|
|
|
|
| 264 |
return mode
|
| 265 |
def set_probe(self,probe):
|
| 266 |
"""Injected by the manager: probe(candidate) -> {'video':..,'audio':..} or None."""
|
|
@@ -348,34 +322,10 @@ class Sender:
|
|
| 348 |
def command(self,destination,candidate=None):
|
| 349 |
# إعلان Enhanced-RTMP يُقاس على الترميز المدفوع فعلًا؛ ملفات الاختبار المحلية لا بروتوكول لها.
|
| 350 |
mode=self.send_mode()
|
| 351 |
-
self.last_send_mode=mode
|
| 352 |
-
if mode=='copy':
|
| 353 |
-
if self.relay_state in ('IDLE','COPY_RECOVERY','REATTACH','ALTERNATE_SOURCE','SOURCE_FAULT',''):
|
| 354 |
-
self.relay_state='COPY_STABLE'
|
| 355 |
-
elif mode=='light':
|
| 356 |
-
self.relay_state='TRANSCODE_STABLE' if self.relay_state in ('TRANSCODE_FALLBACK','TRANSCODE_STABLE') else 'TRANSCODE_FALLBACK'
|
| 357 |
-
if self.transcode_since is None:
|
| 358 |
-
self.transcode_since=time.monotonic()
|
| 359 |
network=destination.startswith(('rtmp://','rtmps://'))
|
| 360 |
enhanced=self.enhanced_advertisement(mode,candidate) if network else []
|
| 361 |
if mode=='copy':
|
| 362 |
video=['-c:v','copy']
|
| 363 |
-
if not self.hevc_payload(candidate):
|
| 364 |
-
video += ['-bsf:v','dump_extra=freq=keyframe']
|
| 365 |
-
elif mode=='light':
|
| 366 |
-
# YouTube 720p30 floor ~1500 kbps + GOP 2s. ultrafast keeps cpu-basic
|
| 367 |
-
# usable; measured fixed720/veryfast/4000k filled 2 cores at 2 publishers.
|
| 368 |
-
width,height,bitrate=720,1280,(os.getenv('YOUTUBE_LIGHT_BITRATE','1800k').strip() or '1800k')
|
| 369 |
-
try:
|
| 370 |
-
bufsize=str(int(str(bitrate).rstrip('kK'))*2)+'k'
|
| 371 |
-
except (TypeError, ValueError):
|
| 372 |
-
bufsize='3600k'
|
| 373 |
-
video=['-vf',self.canvas(width,height),'-c:v','libx264','-preset','ultrafast',
|
| 374 |
-
'-tune','zerolatency','-profile:v','main',
|
| 375 |
-
'-b:v',bitrate,'-maxrate',bitrate,'-bufsize',bufsize,
|
| 376 |
-
'-force_key_frames','expr:gte(t,n_forced*2)',
|
| 377 |
-
'-g','60','-keyint_min','30','-sc_threshold','0',
|
| 378 |
-
'-pix_fmt','yuv420p','-fps_mode','cfr']
|
| 379 |
else:
|
| 380 |
width,height,bitrate=(1080,1920,'6000k') if mode=='fixed1080' else (720,1280,'4000k')
|
| 381 |
# VIDEO_BITRATE was documented in the README but never read anywhere; it now caps the
|
|
@@ -639,8 +589,6 @@ class Sender:
|
|
| 639 |
except Exception as e:
|
| 640 |
self.log(user, '[RELAY] تعذر رصد مصدر بديل دون قطع يوتيوب (%s)' % type(e).__name__)
|
| 641 |
return None
|
| 642 |
-
if not cand or not cand.get('url'):
|
| 643 |
-
cand = getattr(self, 'current_candidate', None)
|
| 644 |
if not cand or cand.get('media') == 'hls' or not cand.get('url'):
|
| 645 |
return None
|
| 646 |
try:
|
|
@@ -658,8 +606,7 @@ class Sender:
|
|
| 658 |
sess['stall_reason'] = ''
|
| 659 |
sess['force_stop'] = False
|
| 660 |
sess['timeouts'] = 0
|
| 661 |
-
|
| 662 |
-
# Resetting last turned a 12s source stall into 8×reattach of a frozen picture.
|
| 663 |
sess['reattaches'] = int(sess.get('reattaches') or 0) + 1
|
| 664 |
self.current_candidate = cand
|
| 665 |
self.set(source=cand.get('label') or cand.get('brief') or '', stalled=False)
|
|
@@ -854,43 +801,19 @@ class Sender:
|
|
| 854 |
# ---- الحارس: لا تعتمد على poll() ولا على رمز الخروج ----
|
| 855 |
stall_limit = self._stall_seconds()
|
| 856 |
progress_limit = max(30.0, float(os.getenv('YOUTUBE_PROGRESS_STALL_SECONDS', '60')))
|
| 857 |
-
# Measured COPY_RECOVERY: origin 1920 copy after light 720 left pid alive,
|
| 858 |
-
# source bytes flowing, out_time=0 for the full 60s progress window.
|
| 859 |
-
# Zero output while the source is fresh is a failed start, not a mid-stream freeze.
|
| 860 |
-
start_limit = max(5.0, float(os.getenv('YOUTUBE_OUTPUT_START_SECONDS', '12')))
|
| 861 |
-
started_at = time.monotonic()
|
| 862 |
next_report = 0.0
|
| 863 |
while not stop.is_set():
|
| 864 |
now = time.monotonic()
|
| 865 |
-
if progress['seconds'] < 1.0 and now - started_at > start_limit:
|
| 866 |
-
src_fresh = (now - (sess['ff_last'] if response is None else sess['last'])) < max(stall_limit or 12.0, 12.0)
|
| 867 |
-
if src_fresh or response is None:
|
| 868 |
-
raise SourceStalled('FFmpeg لم يُخرج إطارًا منذ %.0f ث والمصدر يتدفق (out=0)' % (now - started_at))
|
| 869 |
# الدليل على أن المصدر لم يصمت هو بايتات تيك توك نفسها. تقدّم FFmpeg وحده لا
|
| 870 |
# يكفي: في وضع النسخ يفرّغ المخزن الداخلي لدقائق بعد صمت المصدر، فيبدو البث
|
| 871 |
# حيًا ويوتيوب يبقى على آخر ثانية — وهذا هو التعليق الذي شكا منه المستخدم.
|
| 872 |
# في HLS يستحيل القياس إلا على FFmpeg (هو من يفتح القائمة)، فنقبل تقدّمه.
|
| 873 |
_fresh = (sess['ff_last'] if response is None else sess['last'])
|
| 874 |
-
freeze_limit = max(float(stall_limit or 12.0), 12.0)
|
| 875 |
-
# Frozen picture: out_time stopped. Splicing TikTok into the same FFmpeg
|
| 876 |
-
# does not create IDRs and leaves YouTube on the last frame (measured
|
| 877 |
-
# mrjooj_7: out=123s for 60s, health still good, source_age 20→61).
|
| 878 |
-
if progress['seconds'] >= 1.0 and now - progress['last'] > freeze_limit:
|
| 879 |
-
raise SourceStalled('تجمّد خرج FFmpeg منذ %.0f ث — نعيد copy على نفس حدث يوتيوب' % (now - progress['last']))
|
| 880 |
if stall_limit and not sess.get('swapping') and now - _fresh > stall_limit:
|
| 881 |
-
|
| 882 |
-
|
| 883 |
-
attached=None
|
| 884 |
-
for _try in range(3):
|
| 885 |
-
attached = self._try_reattach(user, p, bridge, sess, stop, refresh,
|
| 886 |
-
read_timeout, retries, response, pump)
|
| 887 |
-
if attached:
|
| 888 |
-
response, pump = attached
|
| 889 |
-
break
|
| 890 |
-
if stop.is_set() or (p.poll() is not None):
|
| 891 |
-
break
|
| 892 |
-
time.sleep(1.0)
|
| 893 |
if attached:
|
|
|
|
| 894 |
continue
|
| 895 |
# نُفيق الخيط أولًا (قد يكون محبوسًا في write على أنبوب ممتلئ)، ثم نرفع:
|
| 896 |
# إغلاق stdin يجعل FFmpeg يرى EOF ويتوقف بلطف بدل SIGKILL.
|
|
@@ -902,22 +825,6 @@ class Sender:
|
|
| 902 |
pass
|
| 903 |
raise SourceStalled('انقطع تدفق المصدر منذ %.0f ث (FFmpeg ما زال حيًا، '
|
| 904 |
'ويوتيوب يبقى على آخر ثانية وصلته)' % (now - _fresh))
|
| 905 |
-
if sess.get('format_changed') and p.poll() is None:
|
| 906 |
-
sess['format_changed'] = False
|
| 907 |
-
self.log(user, '[RELAY] تغيّر شكل المصدر؛ إعادة توصيل تيك توك على نفس FFmpeg — لا قطع يوتيوب')
|
| 908 |
-
attached=None
|
| 909 |
-
for _try in range(3):
|
| 910 |
-
attached = self._try_reattach(user, p, bridge, sess, stop, refresh,
|
| 911 |
-
read_timeout, retries, response, pump)
|
| 912 |
-
if attached:
|
| 913 |
-
response, pump = attached
|
| 914 |
-
break
|
| 915 |
-
if stop.is_set() or p.poll() is not None:
|
| 916 |
-
break
|
| 917 |
-
time.sleep(1.0)
|
| 918 |
-
if attached:
|
| 919 |
-
continue
|
| 920 |
-
raise FormatChanged()
|
| 921 |
rc = p.poll()
|
| 922 |
if rc is not None:
|
| 923 |
if sess['format_changed']:
|
|
|
|
| 100 |
# event. Without this the flap window expires after 10 quiet minutes and the
|
| 101 |
# sender silently drops back to copy mode, restarting the whole failure loop.
|
| 102 |
self.sticky_mode=None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
# Candidate handed to send_mode() so the codec rule can judge the real source.
|
| 104 |
self.current_candidate=None
|
| 105 |
# Journal setting wins over the environment so the mode can be changed from the UI
|
|
|
|
| 117 |
stamp=d.get('last_source_at')
|
| 118 |
if stamp:d['source_age']=round(max(0.0,time.monotonic()-stamp),1)
|
| 119 |
try:
|
| 120 |
+
d['send_mode']=self.sticky_mode or self.mode_override or ''
|
| 121 |
except Exception:
|
| 122 |
d['send_mode']=''
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
return d
|
| 124 |
def wait_until(self,condition,stop,poll=1.0,timeout=None):
|
| 125 |
"""Block while `condition()` holds; returns as soon as it stops or a stop is requested."""
|
|
|
|
| 158 |
fixed720 -> constant 720x1280 canvas, re-encodes (lighter on CPU)
|
| 159 |
"""
|
| 160 |
mode=(self.mode_override or os.getenv('YOUTUBE_SEND_MODE','auto')).strip().lower()
|
| 161 |
+
if mode not in ('auto','copy','fixed1080','fixed720'):
|
| 162 |
+
raise ValueError('وضع الإرسال يجب أن يكون auto أو copy أو fixed1080 أو fixed720')
|
| 163 |
# Sticky transcode is for HEVC / layout flap only. Measured 2026-09-27 on cpu-basic:
|
| 164 |
# two fixed720 publishers used 1.67–1.85 of 2.0 quota (peak 92.6%) with YouTube
|
| 165 |
# health still good — no headroom for a third. If this candidate is H264, copy.
|
| 166 |
if self.sticky_mode:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
cand=self.current_candidate or {}
|
| 168 |
blob=(str(cand.get('codec') or '')+' '+str(cand.get('label') or '')+' '+str(cand.get('brief') or '')).lower()
|
| 169 |
if any(k in blob for k in ('h264','avc')):
|
|
|
|
| 186 |
now=time.monotonic()
|
| 187 |
while flap and now-flap[0]>600:flap.popleft()
|
| 188 |
if len(flap)>=int(os.getenv('YOUTUBE_FLAP_LIMIT','4')):
|
| 189 |
+
mode=os.getenv('YOUTUBE_FLAP_FALLBACK_MODE','fixed720').strip().lower()
|
| 190 |
+
if mode not in ('fixed1080','fixed720'):mode='fixed720'
|
| 191 |
+
self.sticky_mode=mode
|
|
|
|
| 192 |
return mode
|
| 193 |
def note_format_change(self):
|
| 194 |
self.flap.append(time.monotonic())
|
| 195 |
if len(self.flap)>=int(os.getenv('YOUTUBE_FLAP_LIMIT','4')):
|
| 196 |
+
self.log('system','تغيّرت أبعاد المصدر عدة مرات خلال 10 دقائق؛ التحول تلقائيًا إلى كانفس ثابت (%s) لمنع تكرار الانقطاع'%os.getenv('YOUTUBE_FLAP_FALLBACK_MODE','fixed720'))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
@staticmethod
|
| 198 |
def copy_is_safe(declared,probed):
|
| 199 |
"""Stream copy is allowed only on positive proof of H.264, never on absence of evidence.
|
|
|
|
| 229 |
label=str(cand.get('label') or cand.get('brief') or '')
|
| 230 |
if any(k in str(declared).lower() for k in ('h264','avc')) or any(k in label.lower() for k in ('h264','avc')):
|
| 231 |
return None
|
| 232 |
+
mode=os.getenv('YOUTUBE_CODEC_FALLBACK_MODE','fixed720').strip().lower()
|
| 233 |
+
if mode not in ('fixed1080','fixed720'):mode='fixed720'
|
| 234 |
self.sticky_mode=mode
|
| 235 |
why=('ffprobe: '+str(probed)) if probed else (('مُعلَن: '+declared) if declared else 'ترميز مجهول والفحص لم يأتِ بنتيجة')
|
| 236 |
+
self.log('system','لا يمكن نسخ المصدر كما هو ('+why+') — تحويل تلقائي إلى '+mode+
|
| 237 |
+
'؛ هذا يمنع تجمّد الصورة عند المشاهدين عند تغيّر المقاس أو في جولات PK')
|
| 238 |
return mode
|
| 239 |
def set_probe(self,probe):
|
| 240 |
"""Injected by the manager: probe(candidate) -> {'video':..,'audio':..} or None."""
|
|
|
|
| 322 |
def command(self,destination,candidate=None):
|
| 323 |
# إعلان Enhanced-RTMP يُقاس على الترميز المدفوع فعلًا؛ ملفات الاختبار المحلية لا بروتوكول لها.
|
| 324 |
mode=self.send_mode()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
network=destination.startswith(('rtmp://','rtmps://'))
|
| 326 |
enhanced=self.enhanced_advertisement(mode,candidate) if network else []
|
| 327 |
if mode=='copy':
|
| 328 |
video=['-c:v','copy']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 329 |
else:
|
| 330 |
width,height,bitrate=(1080,1920,'6000k') if mode=='fixed1080' else (720,1280,'4000k')
|
| 331 |
# VIDEO_BITRATE was documented in the README but never read anywhere; it now caps the
|
|
|
|
| 589 |
except Exception as e:
|
| 590 |
self.log(user, '[RELAY] تعذر رصد مصدر بديل دون قطع يوتيوب (%s)' % type(e).__name__)
|
| 591 |
return None
|
|
|
|
|
|
|
| 592 |
if not cand or cand.get('media') == 'hls' or not cand.get('url'):
|
| 593 |
return None
|
| 594 |
try:
|
|
|
|
| 606 |
sess['stall_reason'] = ''
|
| 607 |
sess['force_stop'] = False
|
| 608 |
sess['timeouts'] = 0
|
| 609 |
+
sess['last'] = time.monotonic()
|
|
|
|
| 610 |
sess['reattaches'] = int(sess.get('reattaches') or 0) + 1
|
| 611 |
self.current_candidate = cand
|
| 612 |
self.set(source=cand.get('label') or cand.get('brief') or '', stalled=False)
|
|
|
|
| 801 |
# ---- الحارس: لا تعتمد على poll() ولا على رمز الخروج ----
|
| 802 |
stall_limit = self._stall_seconds()
|
| 803 |
progress_limit = max(30.0, float(os.getenv('YOUTUBE_PROGRESS_STALL_SECONDS', '60')))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 804 |
next_report = 0.0
|
| 805 |
while not stop.is_set():
|
| 806 |
now = time.monotonic()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 807 |
# الدليل على أن المصدر لم يصمت هو بايتات تيك توك نفسها. تقدّم FFmpeg وحده لا
|
| 808 |
# يكفي: في وضع النسخ يفرّغ المخزن الداخلي لدقائق بعد صمت المصدر، فيبدو البث
|
| 809 |
# حيًا ويوتيوب يبقى على آخر ثانية — وهذا هو التعليق الذي شكا منه المستخدم.
|
| 810 |
# في HLS يستحيل القياس إلا على FFmpeg (هو من يفتح القائمة)، فنقبل تقدّمه.
|
| 811 |
_fresh = (sess['ff_last'] if response is None else sess['last'])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 812 |
if stall_limit and not sess.get('swapping') and now - _fresh > stall_limit:
|
| 813 |
+
attached = self._try_reattach(user, p, bridge, sess, stop, refresh,
|
| 814 |
+
read_timeout, retries, response, pump)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 815 |
if attached:
|
| 816 |
+
response, pump = attached
|
| 817 |
continue
|
| 818 |
# نُفيق الخيط أولًا (قد يكون محبوسًا في write على أنبوب ممتلئ)، ثم نرفع:
|
| 819 |
# إغلاق stdin يجعل FFmpeg يرى EOF ويتوقف بلطف بدل SIGKILL.
|
|
|
|
| 825 |
pass
|
| 826 |
raise SourceStalled('انقطع تدفق المصدر منذ %.0f ث (FFmpeg ما زال حيًا، '
|
| 827 |
'ويوتيوب يبقى على آخر ثانية وصلته)' % (now - _fresh))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 828 |
rc = p.poll()
|
| 829 |
if rc is not None:
|
| 830 |
if sess['format_changed']:
|
youtube_manager.py
CHANGED
|
@@ -12,7 +12,7 @@ from flv_bridge import FormatChanged
|
|
| 12 |
import inspect
|
| 13 |
|
| 14 |
from youtube_api import GoogleAPI, Journal, APIError
|
| 15 |
-
from ingest import hub, env_rtmp_destination, rtmp_configured, format_session_title, APP_VERSION, max_concurrent_relays
|
| 16 |
|
| 17 |
class ManagedYouTube:
|
| 18 |
def __init__(self,ffmpeg,observe,log,accounts,path,headers_for=None):
|
|
@@ -45,7 +45,7 @@ class ManagedYouTube:
|
|
| 45 |
# إيقاع إعادة المحاولة عندما تكون تيك توك «مباشرة» بلا مرشّحات (لحظات انتقال جولات PK):
|
| 46 |
# كان الحدث ينتظر دورة المشغّل كاملة (30ث) مع كاش رصد قديم، فبدا «معلّقًا على يوتيوب».
|
| 47 |
self.source_retry=max(5,int(os.getenv('YOUTUBE_SOURCE_RETRY',str(min(15,self.scan)))))
|
| 48 |
-
self.max_events=
|
| 49 |
self.last_logged=None;self.status_text='متوقف'
|
| 50 |
# Written in _select_source / _skip_note; declared here so a fresh manager never has to
|
| 51 |
# rely on getattr() defaults (and so the reason survives the first skip).
|
|
@@ -55,7 +55,7 @@ class ManagedYouTube:
|
|
| 55 |
# حتى لو كانت روابطه الموقّعة ما زالت صالحة لساعات. YOUTUBE_OBS_HOLD_SECONDS يضبط
|
| 56 |
# السقف، و0 يطفئ الاحتفاظ.
|
| 57 |
self.hold_seconds=max(0.0,float(os.getenv('YOUTUBE_OBS_HOLD_SECONDS','900')))
|
| 58 |
-
self._held={};self._last_force_at={};self.hammer={};self.retry_at={};self._offline_logged=set();self._gate_note_at={};self.auth_pause={};self.starve={};self._hold_notes={};self.last_seen_status={};self._blocked403={};self._skip_probe_once={};self._skip_hold=set();self._loop_miss={}
|
| 59 |
# هل تقبل دالة الرصد المحقونة force=؟ (app.py يفعّلها؛ الاختبارات قد لا تفعل)
|
| 60 |
try:self._observe_force='force' in inspect.signature(self.observe).parameters
|
| 61 |
except (TypeError,ValueError):self._observe_force=False
|
|
@@ -150,13 +150,9 @@ class ManagedYouTube:
|
|
| 150 |
supported=[c for c in sources if any(k in _cod(c) for k in ('h264','avc','h265','hevc'))] or list(sources)
|
| 151 |
prefer=str(os.getenv('YOUTUBE_PREFER_H264','1')).strip().lower() in ('1','on','true','yes')
|
| 152 |
if prefer and os.getenv('YOUTUBE_SEND_MODE','auto').strip().lower() in ('auto','copy'):
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
if h264:
|
| 157 |
-
supported=sorted(h264, key=lambda c: 0 if str(c.get('media') or '')=='flv' else 1)
|
| 158 |
-
else:
|
| 159 |
-
supported=sorted(supported, key=lambda c: 0 if str(c.get('media') or '')=='flv' else 1)
|
| 160 |
out=supported
|
| 161 |
else:
|
| 162 |
raise RuntimeError('YOUTUBE_VIDEO_MODE يجب أن يكون original أو h264')
|
|
@@ -172,28 +168,12 @@ class ManagedYouTube:
|
|
| 172 |
if w and h:
|
| 173 |
# 720x1440 vertical is 720-class (~1 Mbps), not a 1440p master.
|
| 174 |
return min(w,h)>=1440 or (w*h)>=(1600*900)
|
| 175 |
-
text=str(c.get('label') or c.get('brief') or
|
| 176 |
-
|
| 177 |
-
if 'origin' in text or '1920' in text or '2160' in text or '4k' in text:
|
| 178 |
-
return True
|
| 179 |
-
return False
|
| 180 |
if busy>=2:
|
| 181 |
light=[c for c in out if not _too_heavy(c)]
|
| 182 |
if light:
|
| 183 |
out=light
|
| 184 |
-
def _short(c):
|
| 185 |
-
try:
|
| 186 |
-
return min(int(c.get('width') or 0), int(c.get('height') or 0))
|
| 187 |
-
except (TypeError, ValueError):
|
| 188 |
-
return 0
|
| 189 |
-
# copy-first: never chase a 640/320 PK blip when a 720-class H264 FLV exists.
|
| 190 |
-
hi=[c for c in out if _short(c)>=700]
|
| 191 |
-
if hi:
|
| 192 |
-
out=hi+ [c for c in out if c not in hi]
|
| 193 |
-
out=sorted(out, key=lambda c: (
|
| 194 |
-
0 if str(c.get('media') or '')=='flv' else 1,
|
| 195 |
-
-_short(c),
|
| 196 |
-
))
|
| 197 |
return out[:limit]
|
| 198 |
@staticmethod
|
| 199 |
def _adult_account(user):
|
|
@@ -276,77 +256,20 @@ class ManagedYouTube:
|
|
| 276 |
if self.is_fault_note(ev.get('note')):
|
| 277 |
self.event_update(key, note='')
|
| 278 |
|
| 279 |
-
def _403_is_hard(self,reason):
|
| 280 |
-
s=(reason or '').lower()
|
| 281 |
-
return any(k in s for k in (
|
| 282 |
-
'livestreamingnotenabled','insufficientlivepermissions','livepermissionblocked',
|
| 283 |
-
'youtubesignuprequired','accessnotconfigured','autherror','invalid_grant','forbidden'))
|
| 284 |
-
|
| 285 |
-
def _403_info(self,key):
|
| 286 |
-
info=self._blocked403.get(key)
|
| 287 |
-
if info is None:
|
| 288 |
-
return None
|
| 289 |
-
if isinstance(info,(int,float)):
|
| 290 |
-
return {'t':float(info),'reason':'','hard':True}
|
| 291 |
-
return info
|
| 292 |
-
|
| 293 |
-
def _403_blocked(self,key,user):
|
| 294 |
-
info=self._403_info(key)
|
| 295 |
-
if not info:
|
| 296 |
-
return False
|
| 297 |
-
hard=bool(info.get('hard'))
|
| 298 |
-
limit=float(os.getenv('YOUTUBE_403_BLOCK_SECONDS') or (600 if hard else 90))
|
| 299 |
-
left=int(limit-(time.monotonic()-float(info.get('t') or 0)))
|
| 300 |
-
if left>0:
|
| 301 |
-
why=(info.get('reason') or '?')
|
| 302 |
-
self._skip_note(user,'يوتيوب رفض إنشاء حدث لهذه الجلسة (403/%s)؛ ننتظر %d ث قبل المحاولة من جديد'%(why,left))
|
| 303 |
-
return True
|
| 304 |
-
self._blocked403.pop(key,None)
|
| 305 |
-
return False
|
| 306 |
-
|
| 307 |
-
def _release_soft_403(self,except_user=None):
|
| 308 |
-
"""A finished sibling event may free a YouTube concurrent slot — retry copy, do not transcode."""
|
| 309 |
-
drop=[]
|
| 310 |
-
for k,info in list(self._blocked403.items()):
|
| 311 |
-
meta=info if isinstance(info,dict) else {'hard':True}
|
| 312 |
-
if meta.get('hard'):
|
| 313 |
-
continue
|
| 314 |
-
u=k.split(':',1)[0]
|
| 315 |
-
if except_user and u==except_user:
|
| 316 |
-
continue
|
| 317 |
-
drop.append(k)
|
| 318 |
-
for k in drop:
|
| 319 |
-
self._blocked403.pop(k,None)
|
| 320 |
-
self.log(k.split(':',1)[0],'[MONITOR] أُخلي قفل 403 المؤقت بعد تحرّر مقعد يوتيوب؛ سيُعاد فتح copy')
|
| 321 |
-
|
| 322 |
def _youtube_rejected(self,user,room_id,reason):
|
| 323 |
-
"""N16/N16b: رفض يوتيوب 403 عند إنشاء الحدث أو تحديثه.
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
ح
|
| 327 |
-
"""
|
| 328 |
-
rs=(reason or '').lower()
|
| 329 |
-
if 'ratelimit' in rs or 'userrequestsexceed' in rs:
|
| 330 |
-
self._channel_rate_until=time.monotonic()+180
|
| 331 |
-
self.log(user,'[MONITOR] 403/%s — تهدئة القناة 180ث (API MANAGEMENT LIMITATION)'%reason)
|
| 332 |
-
hard=self._403_is_hard(reason)
|
| 333 |
-
keep=('ready','testing','live','liveStarting')
|
| 334 |
_keys=[k for k,ev in self.journal.read()['events'].items() if ev.get('username')==user and not ev.get('closing')]
|
| 335 |
for _k in _keys:
|
| 336 |
-
ev=self.journal.read()['events'].get(_k) or {}
|
| 337 |
-
# Soft 403 (quota/rateLimit) must not tear down a bound event that already
|
| 338 |
-
# needs FFmpeg. Finish only planned/unbound shells that cannot ingest.
|
| 339 |
-
if (not hard) and ev.get('bound') and ev.get('broadcast_id') and str(ev.get('youtube_status') or '') in keep:
|
| 340 |
-
self.log(user,'[MONITOR] 403/%s على إنشاء جديد؛ نبقي الحدث القائم %s والحالة %s'%(reason, ev.get('broadcast_id'), ev.get('youtube_status')))
|
| 341 |
-
continue
|
| 342 |
self.event_note(_k,'يوتيوب رفض إنشاء/تحديث الحدث (%s): تحقّق من صلاحية البث المباشر وسقف الأحداث في حسابك' % reason)
|
| 343 |
try:self.finish_event(_k)
|
| 344 |
except Exception:self.archive_event(_k,'local_cancelled_'+str(reason))
|
| 345 |
_rid=str(room_id or '')
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
self._blocked403[user+':'+_rid]={'t':time.monotonic(),'reason':str(reason or ''),'hard':hard}
|
| 349 |
-
self.log(user,'[MONITOR] يوتيوب رفض إنشاء الحدث (403/%s hard=%s)؛ قفل %d ث — copy سيُعاد بعد النافذة أو بعد تحرّر مقعد'%(reason,hard,int(limit)))
|
| 350 |
|
| 351 |
# ---------- channel / settings ----------
|
| 352 |
def check_channel(self,allow_switch=False,expected=None,report_only=False):
|
|
@@ -393,7 +316,7 @@ class ManagedYouTube:
|
|
| 393 |
with self.operations:
|
| 394 |
d=self.journal.read()
|
| 395 |
clean={k:str(values.get(k,'')).strip() for k in ('name','privacy','title','description','send_mode')}
|
| 396 |
-
if clean['send_mode'] and clean['send_mode'] not in ('auto','copy','
|
| 397 |
raise ValueError('وضع الإرسال يجب أن يكون auto أو copy أو fixed1080 أو fixed720')
|
| 398 |
if clean['privacy'] not in ('public','unlisted','private'):raise ValueError('خصوصية غير صالحة')
|
| 399 |
if 'send_mode' in values:
|
|
@@ -571,31 +494,14 @@ class ManagedYouTube:
|
|
| 571 |
|
| 572 |
# ---------- events ----------
|
| 573 |
def open_event(self,user,observation):
|
| 574 |
-
with self.operations:
|
| 575 |
-
return self._open_event_body(user,observation)
|
| 576 |
-
def _open_event_body(self,user,observation):
|
| 577 |
d=self.journal.read();key=user+':'+observation['room_id']
|
| 578 |
# قفل رفض 403 يُفحص أولًا: بعد الإغلاق يبقى المفتاح في history والفحص اللاحق كان
|
| 579 |
# يسمح بلوغ الحدث مرة أخرى كل دورة رفض متكررة (الرفض حصل لأن المفتاح في history).
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
left=int(self._channel_rate_until-time.monotonic())
|
| 584 |
-
self._skip_note(user,'يوتيوب 403/rateLimit على القناة؛ انتظار %dث قبل إنشاء حدث جديد'%left)
|
| 585 |
return False
|
| 586 |
-
if key in d['events']:
|
| 587 |
-
self._skip_note(user,'نفس الجلسة في الأحداث')
|
| 588 |
-
return False
|
| 589 |
-
hist=[h for h in d['history'] if h.get('session_key')==key]
|
| 590 |
-
if hist:
|
| 591 |
-
entry=hist[0]
|
| 592 |
-
if entry.get('broadcast_id') and not self._republish_allowed(key,entry):
|
| 593 |
-
if self._resume_archived_event(user,observation,entry):
|
| 594 |
-
return True
|
| 595 |
-
def _drop(dd):
|
| 596 |
-
dd['history']=[h for h in dd['history'] if h.get('session_key')!=key]
|
| 597 |
-
self.journal.change(_drop)
|
| 598 |
-
d=self.journal.read()
|
| 599 |
dt=datetime.now(ZoneInfo('Asia/Baghdad'));settings=d['settings']
|
| 600 |
nickname=str(observation.get('nickname') or '').strip()
|
| 601 |
live_title=str(observation.get('live_title') or '').strip()
|
|
@@ -623,15 +529,9 @@ class ManagedYouTube:
|
|
| 623 |
if not ev['broadcast_id']:
|
| 624 |
if ev['create_pending']:
|
| 625 |
found=self.reconcile('liveBroadcasts',ev['marker'])
|
| 626 |
-
if found:
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
self.log(ev.get('username') or key,'[RELAY] create_pending بلا نتيجة >20ث؛ نعيد محاولة الإنشاء')
|
| 630 |
-
self.event_update(key,create_pending=False,phase='planned')
|
| 631 |
-
else:
|
| 632 |
-
raise RuntimeError('إنشاء حدث سابق غير محسوم؛ لا نكرر POST لمنع التراكم. يلزم مراجعة إذا استمر')
|
| 633 |
-
ev=self.journal.read()['events'][key]
|
| 634 |
-
if not ev.get('broadcast_id'):
|
| 635 |
self.event_update(key,create_pending=True,phase='creating')
|
| 636 |
body={'snippet':{'title':ev['title'],'description':ev['description'],
|
| 637 |
'scheduledStartTime':(datetime.now(timezone.utc)+timedelta(seconds=30)).isoformat()},
|
|
@@ -670,13 +570,16 @@ class ManagedYouTube:
|
|
| 670 |
def finish_event(self,key):
|
| 671 |
ev=self.journal.read()['events'].get(key)
|
| 672 |
if not ev:return True
|
| 673 |
-
self._release_soft_403(except_user=ev.get('username'))
|
| 674 |
self.event_update(key,closing=True,phase='closing')
|
| 675 |
if not self.stop_sender(key):return False
|
| 676 |
ev=self.journal.read()['events'].get(key)
|
| 677 |
if not ev:return True
|
| 678 |
if not ev['broadcast_id']:
|
| 679 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 680 |
ev=self.journal.read()['events'][key]
|
| 681 |
remote=self.remote('liveBroadcasts',ev['broadcast_id'])
|
| 682 |
if not remote:self.archive_event(key,'missing');return True
|
|
@@ -733,25 +636,11 @@ class ManagedYouTube:
|
|
| 733 |
self.log(user or 'system', '[WATCHDOG] أُلغي تأجيل إعادة الاتصال (%d حدث) — %s' % (dropped, why))
|
| 734 |
return dropped
|
| 735 |
|
| 736 |
-
|
| 737 |
-
def _yt_public(self, ev):
|
| 738 |
-
"""Viewers can already open the watch URL — any delay is a YouTube loading spinner."""
|
| 739 |
-
if not ev:
|
| 740 |
-
return False
|
| 741 |
-
if ev.get('was_live'):
|
| 742 |
-
return True
|
| 743 |
-
return str(ev.get('youtube_status') or '') in ('live', 'testing', 'liveStarting')
|
| 744 |
def start_sender(self,key):
|
| 745 |
-
|
| 746 |
-
|
| 747 |
-
#
|
| 748 |
-
# بعد live أي تأخير = دائرة تحميل أمام المشاهد — نلغي القفل ونعيد copy فورًا.
|
| 749 |
until=self.retry_at.get(key)
|
| 750 |
-
if public:
|
| 751 |
-
if until:
|
| 752 |
-
self.retry_at.pop(key, None)
|
| 753 |
-
self._gate_note_at.pop(key, None)
|
| 754 |
-
until=None
|
| 755 |
if until and time.monotonic()<until:
|
| 756 |
left=int(round(until-time.monotonic()))
|
| 757 |
# الصمت هنا هو بالضبط «بالانتظار بلا سبب» الذي شكا منه المستخدم: القفل صحيح لكن يجب
|
|
@@ -811,20 +700,9 @@ class ManagedYouTube:
|
|
| 811 |
self.event_note(key,'مفتاح RTMPS البيئي مشغول بمذيع آخر؛ لن نخلط بثّين على نفس المفتاح')
|
| 812 |
return
|
| 813 |
if hub.at_capacity(except_user=ev['username']):
|
| 814 |
-
self.log(ev['username'],'[RELAY] @%s waiting: at MAX_CONCURRENT_RELAYS=%
|
| 815 |
return
|
| 816 |
if hub.busy(ev['username']):
|
| 817 |
-
slot=hub.snapshot(ev['username']) or {}
|
| 818 |
-
snd=slot.get('sender')
|
| 819 |
-
try:
|
| 820 |
-
ff=bool(snd and snd.alive())
|
| 821 |
-
except Exception:
|
| 822 |
-
ff=False
|
| 823 |
-
if ff:
|
| 824 |
-
return
|
| 825 |
-
hub.release(ev['username'])
|
| 826 |
-
self.log(ev['username'],'[RELAY] أُخلي مقعد مشرف بلا FFmpeg حتى يبدأ copy')
|
| 827 |
-
if (not public) and time.monotonic()-float(self._spawn_at.get(ev['username']) or 0)<8:
|
| 828 |
return
|
| 829 |
# لا نلمس entry هنا: بعد موت خيط الإرسال تُبقي tick() القيد الميت في self.senders،
|
| 830 |
# و`entry['candidate']=None` على قيد ميت كان يرمي TypeError عند كل دورة مشغّل
|
|
@@ -836,12 +714,6 @@ class ManagedYouTube:
|
|
| 836 |
# حين تبدأ الحافة بقبولنا فعلًا يُمسح العقاب وأثره — لا يُتركان حتى أول انقطاع لاحق
|
| 837 |
sender.on_stable=lambda u=ev['username']: self.mark_stream_stable(u)
|
| 838 |
sender.mode_override=self.send_mode_setting() or None
|
| 839 |
-
if ev.get('force_light'):
|
| 840 |
-
cap=max(0,int(os.getenv('YOUTUBE_LIGHT_MAX','1')))
|
| 841 |
-
if cap>0 and self._light_count()<cap:
|
| 842 |
-
sender.sticky_mode='light'
|
| 843 |
-
else:
|
| 844 |
-
self.log(ev['username'],'[RELAY] سقف الترميز الخفيف ممتلئ عند الإقلاع؛ نُبقي copy')
|
| 845 |
thread=threading.Thread(target=self.sender_loop,
|
| 846 |
args=(key,ev['username'],ev['room_id'],destination,stop,sender),daemon=True)
|
| 847 |
self.senders[key]={'thread':thread,'stop':stop,'sender':sender}
|
|
@@ -849,7 +721,6 @@ class ManagedYouTube:
|
|
| 849 |
self.senders.pop(key,None)
|
| 850 |
self.log(ev['username'],'[RELAY] @%s skipped duplicate session'%ev['username'])
|
| 851 |
return
|
| 852 |
-
self._spawn_at[ev['username']]=time.monotonic()
|
| 853 |
self.log(ev['username'], '[RESTART] جلسة إرسال جديدة للحساب (خيط إشراف أُقلع من الصفر)')
|
| 854 |
thread.start()
|
| 855 |
def restart_dead_senders(self):
|
|
@@ -876,8 +747,7 @@ class ManagedYouTube:
|
|
| 876 |
if not since:
|
| 877 |
entry['idle_since']=time.monotonic()
|
| 878 |
continue
|
| 879 |
-
|
| 880 |
-
if time.monotonic()-since<grace:
|
| 881 |
continue
|
| 882 |
self.log(ev.get('username') or key,
|
| 883 |
'[RELAY] @%s sender thread without FFmpeg; recycle'%ev.get('username'))
|
|
@@ -975,16 +845,12 @@ class ManagedYouTube:
|
|
| 975 |
# الوجهة تُقرأ في كل دورة: لو تغيّر مفتاح الاستقبال (إعادة استخدام المصدر أو
|
| 976 |
# إعادة ربط للحدث) بقي هذا الخيط يطرق الباب القديم بمفتاح قديم إلى الأبد.
|
| 977 |
destination = self.destinations.get(user) or destination
|
| 978 |
-
if getattr(sender,'sticky_mode',None)=='light' and self._light_count(except_sender=sender)>=max(0,int(os.getenv('YOUTUBE_LIGHT_MAX','1'))):
|
| 979 |
-
sender.sticky_mode=None
|
| 980 |
-
self.log(user,'[RELAY] سقف الترميز الخفيف ممتلئ؛ هذا المذيع يبقى copy')
|
| 981 |
try:
|
| 982 |
self.log(user,'[RELAY] @%s starting (%s)'%(user,self._source_brief(candidate)))
|
| 983 |
if sender.send(user,candidate,destination,stop, refresh=lambda: self._refresh_flv(user)):sent=True
|
| 984 |
except RelayStartError as e:
|
| 985 |
self.log(user,'[RELAY] @%s start failed; trying fallback (%s)'%(user,self.safe_error(e)[:90]))
|
| 986 |
if candidate is not None:self.mark_unstable(user,candidate,'start failed')
|
| 987 |
-
self._copy_recovery_miss(key,user,'start failed')
|
| 988 |
continue
|
| 989 |
except SourceStalled as e:
|
| 990 |
# Signed URLs in the hold cache are stale once bytes stop. Drop the
|
|
@@ -1004,7 +870,6 @@ class ManagedYouTube:
|
|
| 1004 |
stalled=True
|
| 1005 |
self._skip_probe_once[user]=True
|
| 1006 |
self.retry[user]=self.retry_min
|
| 1007 |
-
self._copy_recovery_miss(key,user,'stalled out=0')
|
| 1008 |
break
|
| 1009 |
except FormatChanged:
|
| 1010 |
# Source resolution/layout changed (PK round started or ended, app
|
|
@@ -1029,11 +894,7 @@ class ManagedYouTube:
|
|
| 1029 |
# Skip the idle wait right after a resolution change; otherwise keep polling.
|
| 1030 |
if restart_at and time.monotonic()-restart_at<30:restart_at=0.0;continue
|
| 1031 |
gap=float(getattr(sender,'last_source_gap',0) or 0)
|
| 1032 |
-
|
| 1033 |
-
if self._yt_public(ev_now):
|
| 1034 |
-
wait=0.5
|
| 1035 |
-
else:
|
| 1036 |
-
wait=0.5 if gap>=8 else self._backoff(user)
|
| 1037 |
if stop.wait(wait):return
|
| 1038 |
def mark_stream_stable(self, user):
|
| 1039 |
"""الحافة تقبل التدفق: تُرمى نافذة العقاب وملاحظتها وعدّادها. تُستدعى من المرسل عند أول
|
|
@@ -1055,16 +916,6 @@ class ManagedYouTube:
|
|
| 1055 |
cleared += 1
|
| 1056 |
if self.hammer.pop(user, None) is not None:
|
| 1057 |
cleared += 1
|
| 1058 |
-
try:
|
| 1059 |
-
for e in (self.senders or {}).values():
|
| 1060 |
-
snd=e.get('sender')
|
| 1061 |
-
cand=getattr(snd,'current_candidate',None) if snd else None
|
| 1062 |
-
if cand and cand.get('url'):
|
| 1063 |
-
self._good_copy[user]=dict(cand)
|
| 1064 |
-
self._recover_copy.discard(user)
|
| 1065 |
-
break
|
| 1066 |
-
except Exception:
|
| 1067 |
-
pass
|
| 1068 |
if cleared:
|
| 1069 |
self.retry[user] = self.retry_min
|
| 1070 |
self.log(user, '[WATCHDOG] الاستقبال يقبل التدفق؛ مُسح تأجيل العقاب وأثره من اللوحة (النوافذ: %d)' % cleared)
|
|
@@ -1129,21 +980,15 @@ class ManagedYouTube:
|
|
| 1129 |
except Exception as e:
|
| 1130 |
self.log(user, '[WATCHDOG] تعذر تجديد عنوان الاستقبال: ' + self.safe_error(e)[:120])
|
| 1131 |
if n >= 3:
|
| 1132 |
-
|
| 1133 |
-
|
| 1134 |
-
|
| 1135 |
-
|
| 1136 |
-
|
| 1137 |
-
self.
|
| 1138 |
-
|
| 1139 |
-
|
| 1140 |
-
|
| 1141 |
-
self.retry[user] = max(self.retry.get(user, 0) or 0, wait)
|
| 1142 |
-
for k in [k for k, ev in self.journal.read()['events'].items() if ev.get('username') == user]:
|
| 1143 |
-
self.retry_at[k] = time.monotonic() + wait
|
| 1144 |
-
self.log(user, '[WATCHDOG] الوجهة أغلقت الاتصال بعد %.0f ث (رمز %s) — المرة %d: تأجيل إعادة الاتصال إلى %d ث بدل النقر على حافة ترفض%s'
|
| 1145 |
-
% (lasted, code, n, int(self._backoff(user)),
|
| 1146 |
-
'؛ السبب المعلوم: الحدث لم يُشغَّل بعد' if not_started else ''))
|
| 1147 |
return
|
| 1148 |
|
| 1149 |
def _backoff(self,user):
|
|
@@ -1224,25 +1069,10 @@ class ManagedYouTube:
|
|
| 1224 |
return None
|
| 1225 |
if (obs or {}).get('status')!='live':
|
| 1226 |
return None
|
| 1227 |
-
good=getattr(self,'_good_copy',{}).get(user) or {}
|
| 1228 |
-
try:
|
| 1229 |
-
gh=min(int(good.get('width') or 0), int(good.get('height') or 0))
|
| 1230 |
-
except (TypeError, ValueError):
|
| 1231 |
-
gh=0
|
| 1232 |
-
fallback=None
|
| 1233 |
for c in self._select_source(user, obs) or []:
|
| 1234 |
-
if c.get('media')=
|
| 1235 |
-
|
| 1236 |
-
|
| 1237 |
-
sh=min(int(c.get('width') or 0), int(c.get('height') or 0))
|
| 1238 |
-
except (TypeError, ValueError):
|
| 1239 |
-
sh=0
|
| 1240 |
-
if gh>=700 and sh and sh<500:
|
| 1241 |
-
if fallback is None:
|
| 1242 |
-
fallback=c
|
| 1243 |
-
continue
|
| 1244 |
-
return c
|
| 1245 |
-
return fallback
|
| 1246 |
def _select_source(self,user,obs):
|
| 1247 |
"""All relayable renditions, stable first. Never drop a working lower-quality source."""
|
| 1248 |
candidates=self.video_sources(obs.get('sources') or [])
|
|
@@ -1262,23 +1092,6 @@ class ManagedYouTube:
|
|
| 1262 |
preferred.insert(0, _alt)
|
| 1263 |
self.log(user, '[WATCHDOG] @%s FLV starved; trying fallback %s'%(user,str(_alt.get('media') or 'alt').upper()))
|
| 1264 |
ordered=preferred+demoted
|
| 1265 |
-
if user in getattr(self,'_recover_copy',set()):
|
| 1266 |
-
def _shortside(c):
|
| 1267 |
-
try:return min(int(c.get('width') or 0), int(c.get('height') or 0))
|
| 1268 |
-
except (TypeError, ValueError):return 0
|
| 1269 |
-
def _720(c):
|
| 1270 |
-
s=_shortside(c)
|
| 1271 |
-
if 700<=s<=900:return True
|
| 1272 |
-
lab=str(c.get('label') or c.get('brief') or c.get('quality_name') or '').lower()
|
| 1273 |
-
if 'origin' in lab or '1920' in lab:return False
|
| 1274 |
-
return 'hd' in lab and 'h264' in lab
|
| 1275 |
-
ranked=[c for c in ordered if _720(c)]
|
| 1276 |
-
rest=[c for c in ordered if c not in ranked]
|
| 1277 |
-
if ranked:
|
| 1278 |
-
ordered=ranked+rest
|
| 1279 |
-
self.log(user,'[SOURCE] @%s COPY_RECOVERY: %d مرشح فئة 720 قبل origin'%(user,len(ranked)))
|
| 1280 |
-
else:
|
| 1281 |
-
self.log(user,'[SOURCE] @%s COPY_RECOVERY بلا فئة 720 — نُبقي الترتيب ونُمهل out=0 12ث'%user)
|
| 1282 |
self.log(user,'[SOURCE] @%s selecting source (%d candidates)'%(user,len(ordered)))
|
| 1283 |
if self._skip_probe_once.pop(user, False):
|
| 1284 |
# After a stall, HTTP probe of every rendition took ~15s of relays=0 (measured).
|
|
@@ -1416,15 +1229,7 @@ class ManagedYouTube:
|
|
| 1416 |
return False
|
| 1417 |
if any(e.get('username')==user for e in d['events'].values()):
|
| 1418 |
return False
|
| 1419 |
-
if
|
| 1420 |
-
self._skip_note(user,'انتظار اكتمال إنشاء حدث يوتيوب الحالي قبل فتح مقعد تالٍ')
|
| 1421 |
-
return False
|
| 1422 |
-
if getattr(self,'_channel_rate_until',0) and time.monotonic()<self._channel_rate_until:
|
| 1423 |
-
left=int(self._channel_rate_until-time.monotonic())
|
| 1424 |
-
self._skip_note(user,'يوتيوب 403/rateLimit على القناة؛ انتظار %dث'%left)
|
| 1425 |
-
return False
|
| 1426 |
-
if self.max_events>0 and len(d['events'])>=self.max_events:
|
| 1427 |
-
self._skip_note(user,'سقف أحداث يوتيوب ممتلئ (%d)'%self.max_events)
|
| 1428 |
return False
|
| 1429 |
key=user+':'+str(obs['room_id'])
|
| 1430 |
entry=next((h for h in d['history'] if h.get('session_key')==key),None)
|
|
@@ -1516,28 +1321,11 @@ class ManagedYouTube:
|
|
| 1516 |
if not d['enabled']:
|
| 1517 |
self.set_status('تعطيل الأتمتة وإنهاء الأحداث الجارية' if d['events'] else 'الأتمتة معطلة')
|
| 1518 |
return
|
| 1519 |
-
# Drop shells that never got a YouTube id — they occupy the cap with no FFmpeg
|
| 1520 |
-
# (measured On: events=1 senders=0 while 4 TikTok LIVE).
|
| 1521 |
-
now_ts=time.time()
|
| 1522 |
-
for key,ev in list(d['events'].items()):
|
| 1523 |
-
if ev.get('broadcast_id') or ev.get('closing') or ev.get('create_pending'):
|
| 1524 |
-
continue
|
| 1525 |
-
if now_ts-float(ev.get('created_at') or 0)>=25:
|
| 1526 |
-
self.log(ev.get('username') or key,'[RELAY] حدث بلا broadcast_id >25ث؛ أرشفة مباشرة لفتح مقعد copy')
|
| 1527 |
-
try:self.archive_event(key,'stale_planned')
|
| 1528 |
-
except Exception:
|
| 1529 |
-
pass
|
| 1530 |
-
d=self.journal.read()
|
| 1531 |
# 2) Discover new live sessions, one event per account, up to the concurrency cap.
|
| 1532 |
cap=self.max_events;active=len(d['events']);capped=False
|
| 1533 |
for user in accounts:
|
| 1534 |
-
if
|
| 1535 |
if any(e['username']==user for e in d['events'].values()):continue
|
| 1536 |
-
if any(not e.get('broadcast_id') for e in d['events'].values()):
|
| 1537 |
-
self._skip_note(user,'انتظار اكتمال إنشاء حدث يوتيوب الحالي قبل فتح مقعد تالٍ')
|
| 1538 |
-
break
|
| 1539 |
-
if getattr(self,'_channel_rate_until',0) and time.monotonic()<self._channel_rate_until:
|
| 1540 |
-
break
|
| 1541 |
# رفض الهوية (توكن منتهٍ) لا يُعالَج بوتيرة الرصد: كل دورة كانت تضرب تيك توك
|
| 1542 |
# برفض مؤكد (~10/د على خمسة حسابات) وتُغرق السجل حتى وهو مكبوت. إيقاف لكل
|
| 1543 |
# حساب بمقدار YOUTUBE_AUTH_PAUSE_SECONDS، يُمسح فورًا عند أول استجابة ليست
|
|
@@ -1672,30 +1460,6 @@ class ManagedYouTube:
|
|
| 1672 |
ev=self.journal.read()['events'].get(key)
|
| 1673 |
if not ev or ev.get('closing'):return
|
| 1674 |
user=ev['username']
|
| 1675 |
-
# Bind/create YouTube BEFORE TikTok observe. Manager observe was offline while
|
| 1676 |
-
# /health showed 4 LIVE → event stuck phase=planned, no broadcast_id, no FFmpeg.
|
| 1677 |
-
if not ev.get('broadcast_id') or not ev.get('bound'):
|
| 1678 |
-
try:
|
| 1679 |
-
if user not in self.destinations:
|
| 1680 |
-
self.ensure_stream(user)
|
| 1681 |
-
remote=self.ensure_broadcast(key)
|
| 1682 |
-
if not remote:
|
| 1683 |
-
return
|
| 1684 |
-
except Exception as e:
|
| 1685 |
-
from youtube_api import APIError
|
| 1686 |
-
if isinstance(e,APIError) and getattr(e,'status',0)==403:
|
| 1687 |
-
self._youtube_rejected(user, ev.get('room_id'), getattr(e,'reason',''))
|
| 1688 |
-
return
|
| 1689 |
-
self._event_error(key,e)
|
| 1690 |
-
evf=self.journal.read()['events'].get(key) or {}
|
| 1691 |
-
if evf and not evf.get('broadcast_id') and time.time()-float(evf.get('created_at') or 0)>=25:
|
| 1692 |
-
try:self.archive_event(key,'stale_planned')
|
| 1693 |
-
except Exception: pass
|
| 1694 |
-
return
|
| 1695 |
-
ev=self.journal.read()['events'].get(key)
|
| 1696 |
-
if not ev or ev.get('closing'):return
|
| 1697 |
-
if ev.get('bound') and ev.get('broadcast_id'):
|
| 1698 |
-
self.start_sender(key)
|
| 1699 |
attention=self.needs_attention(key)
|
| 1700 |
allow_hold=user not in self._skip_hold
|
| 1701 |
entry=self.senders.get(key)
|
|
@@ -1711,11 +1475,6 @@ class ManagedYouTube:
|
|
| 1711 |
if obs['status']=='offline':
|
| 1712 |
ev=self.journal.read()['events'].get(key)
|
| 1713 |
if not ev:return
|
| 1714 |
-
# Manager TikTok often looks offline from the datacenter while /health is LIVE.
|
| 1715 |
-
# Do not tear down an unbound shell on that lie — it burned the only seat (planned, rel=0).
|
| 1716 |
-
if not ev.get('bound') and not ev.get('was_live'):
|
| 1717 |
-
self.event_note(key,'رصد المشغّل غير مباشر؛ نُبقي الحدث ونعيد الربط (المراقب قد يكون LIVE)')
|
| 1718 |
-
return
|
| 1719 |
count=int(ev.get('offline_count') or 0)+1;since=ev.get('offline_since') or time.time()
|
| 1720 |
self.event_update(key,offline_count=count,offline_since=since)
|
| 1721 |
gone=time.time()-float(since)
|
|
@@ -1834,18 +1593,13 @@ class ManagedYouTube:
|
|
| 1834 |
|
| 1835 |
if life in ('complete','revoked'):
|
| 1836 |
self.event_update(key,closing=True);self.finish_event(key);return
|
| 1837 |
-
if life
|
| 1838 |
-
if
|
| 1839 |
self.event_update(key,was_live=True,phase='live')
|
| 1840 |
self.log(user,'[LIVE] يوتيوب أكد أن الحدث مباشر: '+ev['broadcast_id'])
|
| 1841 |
-
#
|
| 1842 |
-
# spinner: YouTube waits for ingest, we refuse to send. Measured On @a3dam_111
|
| 1843 |
-
# 2026-09-28: youtube_status=liveStarting, relay_pid=None, events=1.
|
| 1844 |
self.start_sender(key)
|
| 1845 |
self.stream_health(key,user,ev)
|
| 1846 |
-
ev=self.journal.read()['events'].get(key) or ev
|
| 1847 |
-
self._ingest_copy_watchdog(key,user,ev)
|
| 1848 |
-
self._output_freeze_watchdog(key,user,ev)
|
| 1849 |
return
|
| 1850 |
|
| 1851 |
# life is expected to be 'ready' or 'testing' for a newly bound event
|
|
@@ -1877,182 +1631,13 @@ class ManagedYouTube:
|
|
| 1877 |
self.event_update(key, note='انتظار تأكيد استقبال يوتيوب قبل بدء الحدث')
|
| 1878 |
elif not ev.get('note'):
|
| 1879 |
self.event_note(key,'انتظار تأكيد استقبال يوتيوب قبل بدء الحدث')
|
| 1880 |
-
|
| 1881 |
-
|
| 1882 |
-
|
| 1883 |
-
|
| 1884 |
-
def _copy_recovery_miss(self, key, user, why):
|
| 1885 |
-
if user not in getattr(self,'_recover_copy',set()):
|
| 1886 |
-
return
|
| 1887 |
-
ev=self.journal.read()['events'].get(key) or {}
|
| 1888 |
-
n=int(ev.get('copy_recovery_fail') or 0)+1
|
| 1889 |
-
self.event_update(key, copy_recovery_fail=n)
|
| 1890 |
-
self.log(user,'[RELAY] COPY_RECOVERY miss ×%d (%s)'%(n, why))
|
| 1891 |
-
if n>=2:
|
| 1892 |
-
self._recover_copy.discard(user)
|
| 1893 |
-
entry=self.senders.get(key) or {}
|
| 1894 |
-
snd=entry.get('sender')
|
| 1895 |
-
if snd and self._maybe_light(user, snd, 'copy recovery out=0 ×%d'%n, key=key):
|
| 1896 |
-
self.log(user,'[RELAY] نعود light على نفس الحدث بعد فشل copy recovery')
|
| 1897 |
-
|
| 1898 |
-
def _maybe_copy_recovery(self, key, user, ev, snd):
|
| 1899 |
-
"""Hysteresis: light → copy only after H264 + YT health good for recovery window.
|
| 1900 |
-
|
| 1901 |
-
First H264 packet is not enough (oscillation). Window default 45s = one fast
|
| 1902 |
-
YouTube health cycle (measured YOUTUBE_HEALTH_FAST_INTERVAL).
|
| 1903 |
-
"""
|
| 1904 |
-
if not snd or getattr(snd,'sticky_mode',None)!='light':
|
| 1905 |
-
if snd: snd.copy_ok_since=None
|
| 1906 |
-
return False
|
| 1907 |
-
health=(ev.get('youtube_health') or getattr(snd,'yt_health','') or '')
|
| 1908 |
-
cand=snd.current_candidate or {}
|
| 1909 |
-
blob=(str(cand.get('codec') or '')+' '+str(cand.get('label') or '')+' '+str(cand.get('brief') or '')).lower()
|
| 1910 |
-
h264=any(k in blob for k in ('h264','avc'))
|
| 1911 |
-
if health not in ('good','ok') or not h264:
|
| 1912 |
-
snd.copy_ok_since=None
|
| 1913 |
-
return False
|
| 1914 |
-
now=time.monotonic()
|
| 1915 |
-
if snd.copy_ok_since is None:
|
| 1916 |
-
snd.copy_ok_since=now
|
| 1917 |
-
snd.relay_state='TRANSCODE_STABLE'
|
| 1918 |
-
return False
|
| 1919 |
-
need=float(os.getenv('YOUTUBE_COPY_RECOVERY_SECONDS','45'))
|
| 1920 |
-
if now-snd.copy_ok_since<need:
|
| 1921 |
-
return False
|
| 1922 |
-
fails=int(ev.get('copy_recovery_fail') or 0)
|
| 1923 |
-
if fails>=2:
|
| 1924 |
-
self.log(user,'[RELAY] COPY_RECOVERY فشل مرتين هذه الجلسة؛ نُبقي light (لا oscillation)')
|
| 1925 |
-
return False
|
| 1926 |
-
snd.sticky_mode=None
|
| 1927 |
-
snd.relay_state='COPY_RECOVERY'
|
| 1928 |
-
snd.transcode_reason=''
|
| 1929 |
-
self._held.pop(user, None)
|
| 1930 |
-
self._skip_hold.add(user)
|
| 1931 |
-
getattr(self,'_recover_copy',None)
|
| 1932 |
-
if not hasattr(self,'_recover_copy'):
|
| 1933 |
-
self._recover_copy=set()
|
| 1934 |
-
self._recover_copy.add(user)
|
| 1935 |
-
self.event_update(key, force_light=False, copy_recovery=True)
|
| 1936 |
-
self.log(user,'[RELAY] H264 مستقر %.0fs وyoutube_health=%s — عودة copy (فئة 720، مصدر منعش) على نفس الحدث'%(need,health))
|
| 1937 |
-
self.stop_sender(key)
|
| 1938 |
-
return True
|
| 1939 |
-
|
| 1940 |
-
def _light_count(self, except_sender=None):
|
| 1941 |
-
n=0
|
| 1942 |
-
for e in (self.senders or {}).values():
|
| 1943 |
-
snd=e.get('sender')
|
| 1944 |
-
if not snd or snd is except_sender:
|
| 1945 |
-
continue
|
| 1946 |
-
try:
|
| 1947 |
-
if snd.sticky_mode=='light' or snd.send_mode()=='light':
|
| 1948 |
-
n+=1
|
| 1949 |
-
except Exception:
|
| 1950 |
-
pass
|
| 1951 |
-
return n
|
| 1952 |
-
|
| 1953 |
-
def _maybe_light(self, user, sender, why, key=None):
|
| 1954 |
-
cap=max(0,int(os.getenv('YOUTUBE_LIGHT_MAX','1')))
|
| 1955 |
-
if cap<=0:
|
| 1956 |
-
return False
|
| 1957 |
-
if self._light_count(except_sender=sender)>=cap:
|
| 1958 |
-
self.log(user,'[RELAY] ترميز خفيف عند السقف (%d)؛ نُبقي copy'%cap)
|
| 1959 |
-
return False
|
| 1960 |
-
sender.sticky_mode='light'
|
| 1961 |
-
sender.relay_state='TRANSCODE_FALLBACK'
|
| 1962 |
-
sender.transcode_reason=str(why or '')[:160]
|
| 1963 |
-
sender.transcode_since=time.monotonic()
|
| 1964 |
-
sender.copy_ok_since=None
|
| 1965 |
-
if key:
|
| 1966 |
-
self.event_update(key, force_light=True)
|
| 1967 |
-
self.log(user,'[RELAY] copy لم يحقق ingest مستقر (%s) — ترميز خفيف ultrafast 1800k GOP 2ث'%why)
|
| 1968 |
-
return True
|
| 1969 |
-
|
| 1970 |
-
def _output_freeze_watchdog(self, key, user, ev):
|
| 1971 |
-
"""YouTube health=good is not a moving picture. Frozen out_time = frozen frame.
|
| 1972 |
-
|
| 1973 |
-
Measured: mrjooj_7 copy out=123s for ~60s, source_age 61s, youtube_health=good.
|
| 1974 |
-
"""
|
| 1975 |
-
entry=self.senders.get(key) or {}
|
| 1976 |
-
snd=entry.get('sender')
|
| 1977 |
-
try:
|
| 1978 |
-
snap=snd.snapshot() if snd else {}
|
| 1979 |
-
out=float((snap or {}).get('output_seconds') or 0)
|
| 1980 |
-
alive=bool(snd and snd.alive())
|
| 1981 |
-
except Exception:
|
| 1982 |
-
return
|
| 1983 |
-
now=time.monotonic()
|
| 1984 |
-
prev=self._last_out.get(key)
|
| 1985 |
-
if out>=1.0 and prev and out<=prev[0]+0.15:
|
| 1986 |
-
if alive and now-prev[1]>=12.0 and self._yt_public(ev):
|
| 1987 |
-
self.log(user,'[WATCHDOG] out_time مجمّد %.0fs (%.1fث) والصحة يوتيوب=%s — إعادة copy نفس الحدث'%(now-prev[1], out, ev.get('youtube_health') or ''))
|
| 1988 |
-
self._last_out.pop(key, None)
|
| 1989 |
-
self.stop_sender(key)
|
| 1990 |
-
return
|
| 1991 |
-
return
|
| 1992 |
-
self._last_out[key]=(out, now)
|
| 1993 |
-
|
| 1994 |
-
def _ingest_copy_watchdog(self,key,user,ev):
|
| 1995 |
-
"""Restart copy onto the same broadcast when YouTube ingest is dead but TikTok still flows.
|
| 1996 |
-
|
| 1997 |
-
FFmpeg alive is not proof of ingest. noData/bad + fresh source_age → restart copy;
|
| 1998 |
-
after ≥2 copy restarts escalate to light (capped). same broadcast_id.
|
| 1999 |
-
"""
|
| 2000 |
-
health=(ev.get('youtube_health') or '')
|
| 2001 |
-
if health in ('good','ok'):
|
| 2002 |
-
self._ingest_bad_since.pop(key,None)
|
| 2003 |
-
entry=self.senders.get(key) or {}
|
| 2004 |
-
snd=entry.get('sender')
|
| 2005 |
-
if snd:
|
| 2006 |
-
snd.yt_health=health
|
| 2007 |
-
self._maybe_copy_recovery(key,user,ev,snd)
|
| 2008 |
-
return
|
| 2009 |
-
if health not in ('noData','bad','noStream'):
|
| 2010 |
-
return
|
| 2011 |
-
if time.time()-float(ev.get('created_at') or 0)<90:
|
| 2012 |
-
return
|
| 2013 |
-
entry=self.senders.get(key) or {}
|
| 2014 |
-
snd=entry.get('sender')
|
| 2015 |
-
try:
|
| 2016 |
-
alive=bool(snd and snd.alive())
|
| 2017 |
-
except Exception:
|
| 2018 |
-
alive=False
|
| 2019 |
-
if not alive:
|
| 2020 |
-
self._ingest_bad_since.pop(key,None)
|
| 2021 |
-
return
|
| 2022 |
-
try:
|
| 2023 |
-
age=(snd.snapshot() or {}).get('source_age')
|
| 2024 |
-
except Exception:
|
| 2025 |
-
age=None
|
| 2026 |
-
if age is None or float(age)>3:
|
| 2027 |
-
return
|
| 2028 |
-
now=time.monotonic()
|
| 2029 |
-
since=self._ingest_bad_since.get(key)
|
| 2030 |
-
if since is None:
|
| 2031 |
-
self._ingest_bad_since[key]=now
|
| 2032 |
-
return
|
| 2033 |
-
wait=float(os.getenv('YOUTUBE_NODATA_RESTART_SECONDS','90'))
|
| 2034 |
-
if now-since<wait:
|
| 2035 |
-
return
|
| 2036 |
-
self._ingest_bad_since[key]=now
|
| 2037 |
-
retries=int(ev.get('copy_retries') or 0)+1
|
| 2038 |
-
self.event_update(key, copy_retries=retries)
|
| 2039 |
-
if retries>=2 and snd:
|
| 2040 |
-
if self._maybe_light(user, snd, 'noData/bad ×%d'%retries, key=key):
|
| 2041 |
-
self.stop_sender(key)
|
| 2042 |
-
return
|
| 2043 |
-
self.log(user,'[WATCHDOG] يوتيوب ingest=%s والمصدر يتدفق (age=%s ث)؛ إعادة copy على نفس الحدث %s'%(health,age,ev.get('broadcast_id') or ''))
|
| 2044 |
-
self.stop_sender(key)
|
| 2045 |
-
|
| 2046 |
def stream_health(self,key,user,ev):
|
| 2047 |
"""Read YouTube's own verdict on the inbound stream. The Live Streaming API reports
|
| 2048 |
exactly why viewers buffer (videoIngestionStarved), why the picture can be rejected
|
| 2049 |
(videoResolutionUnsupported / videoCodec) and why playback stalls (gopSizeLong /
|
| 2050 |
openGop). Surfacing it replaces guesswork with ground truth. Polled on a slower
|
| 2051 |
cadence than the life-cycle check so it stays inside the quota budget."""
|
| 2052 |
-
|
| 2053 |
-
slow=max(60,int(os.getenv('YOUTUBE_HEALTH_INTERVAL','180')))
|
| 2054 |
-
fast=max(30,int(os.getenv('YOUTUBE_HEALTH_FAST_INTERVAL','45')))
|
| 2055 |
-
interval=fast if prev_h in ('','noData','bad','noStream') else slow
|
| 2056 |
if time.time()-float(ev.get('last_health') or 0)<interval:return
|
| 2057 |
stream_id=(self.journal.read()['streams'].get(user) or {}).get('id')
|
| 2058 |
if not stream_id:return
|
|
@@ -2070,46 +1655,21 @@ class ManagedYouTube:
|
|
| 2070 |
self.event_update(key,youtube_health=health.get('status',''),youtube_issues=text,last_issues=text)
|
| 2071 |
if text and text!=prev:
|
| 2072 |
self.log(user,'يوتيوب يشخّص البث: '+text)
|
| 2073 |
-
# GOP too long / starved ingest → YouTube loading. bitrateLow alone is not enough
|
| 2074 |
-
# (measured: 3× H264 copy ~1 Mbps stayed good/ok at ~8% CPU).
|
| 2075 |
-
hard=any(k in (text or '') for k in ('gopSizeLong','openGop','videoIngestionStarved'))
|
| 2076 |
-
entry=self.senders.get(key) or {}
|
| 2077 |
-
snd=entry.get('sender')
|
| 2078 |
-
if snd:
|
| 2079 |
-
snd.yt_health=health.get('status','')
|
| 2080 |
-
if hard:
|
| 2081 |
-
if snd and self._maybe_light(user, snd, text[:80], key=key):
|
| 2082 |
-
self.stop_sender(key)
|
| 2083 |
-
return
|
| 2084 |
-
if snd:
|
| 2085 |
-
self._maybe_copy_recovery(key,user,self.journal.read()['events'].get(key) or ev,snd)
|
| 2086 |
def run_manager(self):
|
| 2087 |
checked=False
|
| 2088 |
last_tick=0.0
|
| 2089 |
try:
|
| 2090 |
-
# Journal force_light survives Space rebuild and started origin 1920 as light
|
| 2091 |
-
# at 90–100% of 2 cores (On 2026-09-28). Copy-first on boot; light only from
|
| 2092 |
-
# a live ingest fault in this process.
|
| 2093 |
-
try:
|
| 2094 |
-
for _k,_ev in list(self.journal.read()['events'].items()):
|
| 2095 |
-
if _ev.get('force_light') or _ev.get('copy_recovery_fail'):
|
| 2096 |
-
self.event_update(_k, force_light=False, copy_recovery_fail=0)
|
| 2097 |
-
except Exception:
|
| 2098 |
-
pass
|
| 2099 |
while not self.manager_stop.is_set():
|
| 2100 |
delay=self.scan
|
| 2101 |
try:
|
| 2102 |
now=time.monotonic()
|
| 2103 |
-
|
| 2104 |
-
|
| 2105 |
-
|
| 2106 |
-
|
| 2107 |
-
|
| 2108 |
-
|
| 2109 |
-
|
| 2110 |
-
last_tick=now
|
| 2111 |
-
else:
|
| 2112 |
-
self.restart_dead_senders()
|
| 2113 |
d=self.journal.read()
|
| 2114 |
if not d['enabled'] and not d['events']:self.set_status('الأتمتة معطلة؛ لا إرسال ولا إنشاء أحداث')
|
| 2115 |
dead=any(not (e.get('thread') and e['thread'].is_alive()) for e in list(self.senders.values()))
|
|
@@ -2120,17 +1680,7 @@ class ManagedYouTube:
|
|
| 2120 |
if not isinstance(e,(APIError,RuntimeError,ValueError)):text+=' ('+type(e).__name__+')'
|
| 2121 |
self.set_status(text)
|
| 2122 |
if text!=self.last_logged:self.log('youtube',text);self.last_logged=text
|
| 2123 |
-
if isinstance(e,APIError) and (e.status in (401,403) or e.reason=='invalid_grant'):delay=300
|
| 2124 |
-
elif isinstance(e,APIError) and e.status==429 and e.reason=='localDailyBudgetReached':
|
| 2125 |
-
# Local meter (not Google). A 300s freeze left 4 TikTok LIVE at rel=0.
|
| 2126 |
-
def _zero(d):
|
| 2127 |
-
b=d.get('budget') or {}
|
| 2128 |
-
b['estimated_units']=0;d['budget']=b
|
| 2129 |
-
try:self.journal.change(_zero)
|
| 2130 |
-
except Exception: pass
|
| 2131 |
-
self.log('youtube','[RELAY] local YouTube API meter was full; cleared so copy can start')
|
| 2132 |
-
delay=8
|
| 2133 |
-
elif isinstance(e,APIError) and e.status in (429,):delay=300
|
| 2134 |
else:delay=60
|
| 2135 |
self.manager_stop.wait(delay)
|
| 2136 |
finally:
|
|
|
|
| 12 |
import inspect
|
| 13 |
|
| 14 |
from youtube_api import GoogleAPI, Journal, APIError
|
| 15 |
+
from ingest import hub, env_rtmp_destination, rtmp_configured, format_session_title, APP_VERSION, max_concurrent_relays
|
| 16 |
|
| 17 |
class ManagedYouTube:
|
| 18 |
def __init__(self,ffmpeg,observe,log,accounts,path,headers_for=None):
|
|
|
|
| 45 |
# إيقاع إعادة المحاولة عندما تكون تيك توك «مباشرة» بلا مرشّحات (لحظات انتقال جولات PK):
|
| 46 |
# كان الحدث ينتظر دورة المشغّل كاملة (30ث) مع كاش رصد قديم، فبدا «معلّقًا على يوتيوب».
|
| 47 |
self.source_retry=max(5,int(os.getenv('YOUTUBE_SOURCE_RETRY',str(min(15,self.scan)))))
|
| 48 |
+
self.max_events=max(1,int(os.getenv('YOUTUBE_MAX_EVENTS') or max_concurrent_relays()))
|
| 49 |
self.last_logged=None;self.status_text='متوقف'
|
| 50 |
# Written in _select_source / _skip_note; declared here so a fresh manager never has to
|
| 51 |
# rely on getattr() defaults (and so the reason survives the first skip).
|
|
|
|
| 55 |
# حتى لو كانت روابطه الموقّعة ما زالت صالحة لساعات. YOUTUBE_OBS_HOLD_SECONDS يضبط
|
| 56 |
# السقف، و0 يطفئ الاحتفاظ.
|
| 57 |
self.hold_seconds=max(0.0,float(os.getenv('YOUTUBE_OBS_HOLD_SECONDS','900')))
|
| 58 |
+
self._held={};self._last_force_at={};self.hammer={};self.retry_at={};self._offline_logged=set();self._gate_note_at={};self.auth_pause={};self.starve={};self._hold_notes={};self.last_seen_status={};self._blocked403={};self._skip_probe_once={};self._skip_hold=set();self._loop_miss={}
|
| 59 |
# هل تقبل دالة الرصد المحقونة force=؟ (app.py يفعّلها؛ الاختبارات قد لا تفعل)
|
| 60 |
try:self._observe_force='force' in inspect.signature(self.observe).parameters
|
| 61 |
except (TypeError,ValueError):self._observe_force=False
|
|
|
|
| 150 |
supported=[c for c in sources if any(k in _cod(c) for k in ('h264','avc','h265','hevc'))] or list(sources)
|
| 151 |
prefer=str(os.getenv('YOUTUBE_PREFER_H264','1')).strip().lower() in ('1','on','true','yes')
|
| 152 |
if prefer and os.getenv('YOUTUBE_SEND_MODE','auto').strip().lower() in ('auto','copy'):
|
| 153 |
+
supported=sorted(supported,key=lambda c:(
|
| 154 |
+
0 if any(k in _cod(c) for k in ('h264','avc')) else 1,
|
| 155 |
+
0 if str(c.get('media') or '')=='flv' else 1))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
out=supported
|
| 157 |
else:
|
| 158 |
raise RuntimeError('YOUTUBE_VIDEO_MODE يجب أن يكون original أو h264')
|
|
|
|
| 168 |
if w and h:
|
| 169 |
# 720x1440 vertical is 720-class (~1 Mbps), not a 1440p master.
|
| 170 |
return min(w,h)>=1440 or (w*h)>=(1600*900)
|
| 171 |
+
text=str(c.get('label') or c.get('brief') or '')
|
| 172 |
+
return any(x in text for x in ('2160','4k','4K'))
|
|
|
|
|
|
|
|
|
|
| 173 |
if busy>=2:
|
| 174 |
light=[c for c in out if not _too_heavy(c)]
|
| 175 |
if light:
|
| 176 |
out=light
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
return out[:limit]
|
| 178 |
@staticmethod
|
| 179 |
def _adult_account(user):
|
|
|
|
| 256 |
if self.is_fault_note(ev.get('note')):
|
| 257 |
self.event_update(key, note='')
|
| 258 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 259 |
def _youtube_rejected(self,user,room_id,reason):
|
| 260 |
+
"""N16/N16b: رفض يوتيوب 403 (صلاحية/سقف) عند إنشاء الحدث أو تحديثه.
|
| 261 |
+
ليست خطأً عابرًا: حدث planned بلا مفتاح يعني مقعدًا محروقًا من السقف وحالة
|
| 262 |
+
مربكة على اللوحة، وإعادة المحاولة كل دورة مسح تكرر الرفض بلا نهاية. نكتب
|
| 263 |
+
ملاحظة صادقة، نُنهي كل أحداث المستخدم غير المغلقة، ونقفل مفتاح الجلسة داخل
|
| 264 |
+
نافذة زمنية — يُعاد الاكتشاف تلقائيًا بعد معالجة الصلاحية أو انقضاء النافذة."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
_keys=[k for k,ev in self.journal.read()['events'].items() if ev.get('username')==user and not ev.get('closing')]
|
| 266 |
for _k in _keys:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 267 |
self.event_note(_k,'يوتيوب رفض إنشاء/تحديث الحدث (%s): تحقّق من صلاحية البث المباشر وسقف الأحداث في حسابك' % reason)
|
| 268 |
try:self.finish_event(_k)
|
| 269 |
except Exception:self.archive_event(_k,'local_cancelled_'+str(reason))
|
| 270 |
_rid=str(room_id or '')
|
| 271 |
+
if _rid:self._blocked403[user+':'+_rid]=time.monotonic()
|
| 272 |
+
self.log(user,'[MONITOR] يوتيوب رفض إنشاء الحدث (403/%s)؛ أُغلق حدثه ويُمنع تكرار المفتاح %d ث — يُعاد الاكتشاف بعد معالجة الصلاحية'%(reason,int(float(os.getenv('YOUTUBE_403_BLOCK_SECONDS','600')))))
|
|
|
|
|
|
|
| 273 |
|
| 274 |
# ---------- channel / settings ----------
|
| 275 |
def check_channel(self,allow_switch=False,expected=None,report_only=False):
|
|
|
|
| 316 |
with self.operations:
|
| 317 |
d=self.journal.read()
|
| 318 |
clean={k:str(values.get(k,'')).strip() for k in ('name','privacy','title','description','send_mode')}
|
| 319 |
+
if clean['send_mode'] and clean['send_mode'] not in ('auto','copy','fixed1080','fixed720'):
|
| 320 |
raise ValueError('وضع الإرسال يجب أن يكون auto أو copy أو fixed1080 أو fixed720')
|
| 321 |
if clean['privacy'] not in ('public','unlisted','private'):raise ValueError('خصوصية غير صالحة')
|
| 322 |
if 'send_mode' in values:
|
|
|
|
| 494 |
|
| 495 |
# ---------- events ----------
|
| 496 |
def open_event(self,user,observation):
|
|
|
|
|
|
|
|
|
|
| 497 |
d=self.journal.read();key=user+':'+observation['room_id']
|
| 498 |
# قفل رفض 403 يُفحص أولًا: بعد الإغلاق يبقى المفتاح في history والفحص اللاحق كان
|
| 499 |
# يسمح بلوغ الحدث مرة أخرى كل دورة رفض متكررة (الرفض حصل لأن المفتاح في history).
|
| 500 |
+
block=self._blocked403.get(key)
|
| 501 |
+
if block and time.monotonic()-block<float(os.getenv('YOUTUBE_403_BLOCK_SECONDS','600')):
|
| 502 |
+
self._skip_note(user,'يوتيوب رفض إنشاء حدث لهذه الجلسة (403) مؤخرًا؛ ننتظر %d ث قبل المحاولة من جديد'%int(float(os.getenv('YOUTUBE_403_BLOCK_SECONDS','600'))-(time.monotonic()-block)))
|
|
|
|
|
|
|
| 503 |
return False
|
| 504 |
+
if key in d['events'] or any(h['session_key']==key for h in d['history']):return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 505 |
dt=datetime.now(ZoneInfo('Asia/Baghdad'));settings=d['settings']
|
| 506 |
nickname=str(observation.get('nickname') or '').strip()
|
| 507 |
live_title=str(observation.get('live_title') or '').strip()
|
|
|
|
| 529 |
if not ev['broadcast_id']:
|
| 530 |
if ev['create_pending']:
|
| 531 |
found=self.reconcile('liveBroadcasts',ev['marker'])
|
| 532 |
+
if not found:raise RuntimeError('إنشاء حدث سابق غير محسوم؛ لا نكرر POST لمنع التراكم. يلزم مراجعة إذا استمر')
|
| 533 |
+
self.event_update(key,broadcast_id=found['id'],create_pending=False,phase='created')
|
| 534 |
+
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 535 |
self.event_update(key,create_pending=True,phase='creating')
|
| 536 |
body={'snippet':{'title':ev['title'],'description':ev['description'],
|
| 537 |
'scheduledStartTime':(datetime.now(timezone.utc)+timedelta(seconds=30)).isoformat()},
|
|
|
|
| 570 |
def finish_event(self,key):
|
| 571 |
ev=self.journal.read()['events'].get(key)
|
| 572 |
if not ev:return True
|
|
|
|
| 573 |
self.event_update(key,closing=True,phase='closing')
|
| 574 |
if not self.stop_sender(key):return False
|
| 575 |
ev=self.journal.read()['events'].get(key)
|
| 576 |
if not ev:return True
|
| 577 |
if not ev['broadcast_id']:
|
| 578 |
+
if ev['create_pending']:
|
| 579 |
+
found=self.reconcile('liveBroadcasts',ev['marker'])
|
| 580 |
+
if not found:raise RuntimeError('حدث غير محسوم عند الإنهاء؛ لا ننشئ أو نحذف بالتخمين')
|
| 581 |
+
self.event_update(key,broadcast_id=found['id'],create_pending=False)
|
| 582 |
+
else:self.archive_event(key,'cancelled_before_create');return True
|
| 583 |
ev=self.journal.read()['events'][key]
|
| 584 |
remote=self.remote('liveBroadcasts',ev['broadcast_id'])
|
| 585 |
if not remote:self.archive_event(key,'missing');return True
|
|
|
|
| 636 |
self.log(user or 'system', '[WATCHDOG] أُلغي تأجيل إعادة الاتصال (%d حدث) — %s' % (dropped, why))
|
| 637 |
return dropped
|
| 638 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 639 |
def start_sender(self,key):
|
| 640 |
+
# عقاب الوتيرة يجب أن يصمد أمام نقرات tick(): أي أن إعادة الاتصال المؤجّلة بسبب رفض
|
| 641 |
+
# الوجهة (187) لا تُبطلها دورة رصد جاءت بعد 15 ث. بدون هذا القفل يبقى التأجيل شكليًا
|
| 642 |
+
# لأن tick/drive_event يستدعيان start_sender كل دورة.
|
|
|
|
| 643 |
until=self.retry_at.get(key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 644 |
if until and time.monotonic()<until:
|
| 645 |
left=int(round(until-time.monotonic()))
|
| 646 |
# الصمت هنا هو بالضبط «بالانتظار بلا سبب» الذي شكا منه المستخدم: القفل صحيح لكن يجب
|
|
|
|
| 700 |
self.event_note(key,'مفتاح RTMPS البيئي مشغول بمذيع آخر؛ لن نخلط بثّين على نفس المفتاح')
|
| 701 |
return
|
| 702 |
if hub.at_capacity(except_user=ev['username']):
|
| 703 |
+
self.log(ev['username'],'[RELAY] @%s waiting: at MAX_CONCURRENT_RELAYS=%d'%(ev['username'], max_concurrent_relays()))
|
| 704 |
return
|
| 705 |
if hub.busy(ev['username']):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 706 |
return
|
| 707 |
# لا نلمس entry هنا: بعد موت خيط الإرسال تُبقي tick() القيد الميت في self.senders،
|
| 708 |
# و`entry['candidate']=None` على قيد ميت كان يرمي TypeError عند كل دورة مشغّل
|
|
|
|
| 714 |
# حين تبدأ الحافة بقبولنا فعلًا يُمسح العقاب وأثره — لا يُتركان حتى أول انقطاع لاحق
|
| 715 |
sender.on_stable=lambda u=ev['username']: self.mark_stream_stable(u)
|
| 716 |
sender.mode_override=self.send_mode_setting() or None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 717 |
thread=threading.Thread(target=self.sender_loop,
|
| 718 |
args=(key,ev['username'],ev['room_id'],destination,stop,sender),daemon=True)
|
| 719 |
self.senders[key]={'thread':thread,'stop':stop,'sender':sender}
|
|
|
|
| 721 |
self.senders.pop(key,None)
|
| 722 |
self.log(ev['username'],'[RELAY] @%s skipped duplicate session'%ev['username'])
|
| 723 |
return
|
|
|
|
| 724 |
self.log(ev['username'], '[RESTART] جلسة إرسال جديدة للحساب (خيط إشراف أُقلع من الصفر)')
|
| 725 |
thread.start()
|
| 726 |
def restart_dead_senders(self):
|
|
|
|
| 747 |
if not since:
|
| 748 |
entry['idle_since']=time.monotonic()
|
| 749 |
continue
|
| 750 |
+
if time.monotonic()-since<20:
|
|
|
|
| 751 |
continue
|
| 752 |
self.log(ev.get('username') or key,
|
| 753 |
'[RELAY] @%s sender thread without FFmpeg; recycle'%ev.get('username'))
|
|
|
|
| 845 |
# الوجهة تُقرأ في كل دورة: لو تغيّر مفتاح الاستقبال (إعادة استخدام المصدر أو
|
| 846 |
# إعادة ربط للحدث) بقي هذا الخيط يطرق الباب القديم بمفتاح قديم إلى الأبد.
|
| 847 |
destination = self.destinations.get(user) or destination
|
|
|
|
|
|
|
|
|
|
| 848 |
try:
|
| 849 |
self.log(user,'[RELAY] @%s starting (%s)'%(user,self._source_brief(candidate)))
|
| 850 |
if sender.send(user,candidate,destination,stop, refresh=lambda: self._refresh_flv(user)):sent=True
|
| 851 |
except RelayStartError as e:
|
| 852 |
self.log(user,'[RELAY] @%s start failed; trying fallback (%s)'%(user,self.safe_error(e)[:90]))
|
| 853 |
if candidate is not None:self.mark_unstable(user,candidate,'start failed')
|
|
|
|
| 854 |
continue
|
| 855 |
except SourceStalled as e:
|
| 856 |
# Signed URLs in the hold cache are stale once bytes stop. Drop the
|
|
|
|
| 870 |
stalled=True
|
| 871 |
self._skip_probe_once[user]=True
|
| 872 |
self.retry[user]=self.retry_min
|
|
|
|
| 873 |
break
|
| 874 |
except FormatChanged:
|
| 875 |
# Source resolution/layout changed (PK round started or ended, app
|
|
|
|
| 894 |
# Skip the idle wait right after a resolution change; otherwise keep polling.
|
| 895 |
if restart_at and time.monotonic()-restart_at<30:restart_at=0.0;continue
|
| 896 |
gap=float(getattr(sender,'last_source_gap',0) or 0)
|
| 897 |
+
wait=0.5 if gap>=8 else self._backoff(user)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 898 |
if stop.wait(wait):return
|
| 899 |
def mark_stream_stable(self, user):
|
| 900 |
"""الحافة تقبل التدفق: تُرمى نافذة العقاب وملاحظتها وعدّادها. تُستدعى من المرسل عند أول
|
|
|
|
| 916 |
cleared += 1
|
| 917 |
if self.hammer.pop(user, None) is not None:
|
| 918 |
cleared += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 919 |
if cleared:
|
| 920 |
self.retry[user] = self.retry_min
|
| 921 |
self.log(user, '[WATCHDOG] الاستقبال يقبل التدفق؛ مُسح تأجيل العقاب وأثره من اللوحة (النوافذ: %d)' % cleared)
|
|
|
|
| 980 |
except Exception as e:
|
| 981 |
self.log(user, '[WATCHDOG] تعذر تجديد عنوان الاستقبال: ' + self.safe_error(e)[:120])
|
| 982 |
if n >= 3:
|
| 983 |
+
# أرضية 20 ث × تصاعد هندسي: retry_min=1 في الصورة كان يجعل الطلعة الثالثة «2 ث»
|
| 984 |
+
# (لا شيء يتغيّر فعليًا)، لذا الحدّ الأدنى 20 ث ثم 40 و80 حتى 120.
|
| 985 |
+
wait = min(120.0, max(20.0, self.retry_min) * (2 ** (n - 3)))
|
| 986 |
+
self.retry[user] = max(self.retry.get(user, 0) or 0, wait)
|
| 987 |
+
for k in [k for k, ev in self.journal.read()['events'].items() if ev.get('username') == user]:
|
| 988 |
+
self.retry_at[k] = time.monotonic() + wait # يقفل start_sender من tick() نفسه
|
| 989 |
+
self.log(user, '[WATCHDOG] الوجهة أغلقت الاتصال بعد %.0f ث (رمز %s) — المرة %d: تأجيل إعادة الاتصال إلى %d ث بدل النقر على حافة ترفض%s'
|
| 990 |
+
% (lasted, code, n, int(self._backoff(user)),
|
| 991 |
+
'؛ السبب المعلوم: الحدث لم يُشغَّل بعد' if not_started else ''))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 992 |
return
|
| 993 |
|
| 994 |
def _backoff(self,user):
|
|
|
|
| 1069 |
return None
|
| 1070 |
if (obs or {}).get('status')!='live':
|
| 1071 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1072 |
for c in self._select_source(user, obs) or []:
|
| 1073 |
+
if c.get('media')!='hls' and c.get('url'):
|
| 1074 |
+
return c
|
| 1075 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1076 |
def _select_source(self,user,obs):
|
| 1077 |
"""All relayable renditions, stable first. Never drop a working lower-quality source."""
|
| 1078 |
candidates=self.video_sources(obs.get('sources') or [])
|
|
|
|
| 1092 |
preferred.insert(0, _alt)
|
| 1093 |
self.log(user, '[WATCHDOG] @%s FLV starved; trying fallback %s'%(user,str(_alt.get('media') or 'alt').upper()))
|
| 1094 |
ordered=preferred+demoted
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1095 |
self.log(user,'[SOURCE] @%s selecting source (%d candidates)'%(user,len(ordered)))
|
| 1096 |
if self._skip_probe_once.pop(user, False):
|
| 1097 |
# After a stall, HTTP probe of every rendition took ~15s of relays=0 (measured).
|
|
|
|
| 1229 |
return False
|
| 1230 |
if any(e.get('username')==user for e in d['events'].values()):
|
| 1231 |
return False
|
| 1232 |
+
if len(d['events'])>=self.max_events:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1233 |
return False
|
| 1234 |
key=user+':'+str(obs['room_id'])
|
| 1235 |
entry=next((h for h in d['history'] if h.get('session_key')==key),None)
|
|
|
|
| 1321 |
if not d['enabled']:
|
| 1322 |
self.set_status('تعطيل الأتمتة وإنهاء الأحداث الجارية' if d['events'] else 'الأتمتة معطلة')
|
| 1323 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1324 |
# 2) Discover new live sessions, one event per account, up to the concurrency cap.
|
| 1325 |
cap=self.max_events;active=len(d['events']);capped=False
|
| 1326 |
for user in accounts:
|
| 1327 |
+
if active>=cap:capped=True;break
|
| 1328 |
if any(e['username']==user for e in d['events'].values()):continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1329 |
# رفض الهوية (توكن منتهٍ) لا يُعالَج بوتيرة الرصد: كل دورة كانت تضرب تيك توك
|
| 1330 |
# برفض مؤكد (~10/د على خمسة حسابات) وتُغرق السجل حتى وهو مكبوت. إيقاف لكل
|
| 1331 |
# حساب بمقدار YOUTUBE_AUTH_PAUSE_SECONDS، يُمسح فورًا عند أول استجابة ليست
|
|
|
|
| 1460 |
ev=self.journal.read()['events'].get(key)
|
| 1461 |
if not ev or ev.get('closing'):return
|
| 1462 |
user=ev['username']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1463 |
attention=self.needs_attention(key)
|
| 1464 |
allow_hold=user not in self._skip_hold
|
| 1465 |
entry=self.senders.get(key)
|
|
|
|
| 1475 |
if obs['status']=='offline':
|
| 1476 |
ev=self.journal.read()['events'].get(key)
|
| 1477 |
if not ev:return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1478 |
count=int(ev.get('offline_count') or 0)+1;since=ev.get('offline_since') or time.time()
|
| 1479 |
self.event_update(key,offline_count=count,offline_since=since)
|
| 1480 |
gone=time.time()-float(since)
|
|
|
|
| 1593 |
|
| 1594 |
if life in ('complete','revoked'):
|
| 1595 |
self.event_update(key,closing=True);self.finish_event(key);return
|
| 1596 |
+
if life=='live':
|
| 1597 |
+
if not ev['was_live']:
|
| 1598 |
self.event_update(key,was_live=True,phase='live')
|
| 1599 |
self.log(user,'[LIVE] يوتيوب أكد أن الحدث مباشر: '+ev['broadcast_id'])
|
| 1600 |
+
# Now safe to ensure sender is running; viewer already watching
|
|
|
|
|
|
|
| 1601 |
self.start_sender(key)
|
| 1602 |
self.stream_health(key,user,ev)
|
|
|
|
|
|
|
|
|
|
| 1603 |
return
|
| 1604 |
|
| 1605 |
# life is expected to be 'ready' or 'testing' for a newly bound event
|
|
|
|
| 1631 |
self.event_update(key, note='انتظار تأكيد استقبال يوتيوب قبل بدء الحدث')
|
| 1632 |
elif not ev.get('note'):
|
| 1633 |
self.event_note(key,'انتظار تأكيد استقبال يوتيوب قبل بدء الحدث')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1634 |
def stream_health(self,key,user,ev):
|
| 1635 |
"""Read YouTube's own verdict on the inbound stream. The Live Streaming API reports
|
| 1636 |
exactly why viewers buffer (videoIngestionStarved), why the picture can be rejected
|
| 1637 |
(videoResolutionUnsupported / videoCodec) and why playback stalls (gopSizeLong /
|
| 1638 |
openGop). Surfacing it replaces guesswork with ground truth. Polled on a slower
|
| 1639 |
cadence than the life-cycle check so it stays inside the quota budget."""
|
| 1640 |
+
interval=max(60,int(os.getenv('YOUTUBE_HEALTH_INTERVAL','180')))
|
|
|
|
|
|
|
|
|
|
| 1641 |
if time.time()-float(ev.get('last_health') or 0)<interval:return
|
| 1642 |
stream_id=(self.journal.read()['streams'].get(user) or {}).get('id')
|
| 1643 |
if not stream_id:return
|
|
|
|
| 1655 |
self.event_update(key,youtube_health=health.get('status',''),youtube_issues=text,last_issues=text)
|
| 1656 |
if text and text!=prev:
|
| 1657 |
self.log(user,'يوتيوب يشخّص البث: '+text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1658 |
def run_manager(self):
|
| 1659 |
checked=False
|
| 1660 |
last_tick=0.0
|
| 1661 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1662 |
while not self.manager_stop.is_set():
|
| 1663 |
delay=self.scan
|
| 1664 |
try:
|
| 1665 |
now=time.monotonic()
|
| 1666 |
+
with self.operations:
|
| 1667 |
+
if not checked:self.check_channel();checked=True
|
| 1668 |
+
if now-last_tick>=self.scan:
|
| 1669 |
+
self.tick()
|
| 1670 |
+
last_tick=now
|
| 1671 |
+
else:
|
| 1672 |
+
self.restart_dead_senders()
|
|
|
|
|
|
|
|
|
|
| 1673 |
d=self.journal.read()
|
| 1674 |
if not d['enabled'] and not d['events']:self.set_status('الأتمتة معطلة؛ لا إرسال ولا إنشاء أحداث')
|
| 1675 |
dead=any(not (e.get('thread') and e['thread'].is_alive()) for e in list(self.senders.values()))
|
|
|
|
| 1680 |
if not isinstance(e,(APIError,RuntimeError,ValueError)):text+=' ('+type(e).__name__+')'
|
| 1681 |
self.set_status(text)
|
| 1682 |
if text!=self.last_logged:self.log('youtube',text);self.last_logged=text
|
| 1683 |
+
if isinstance(e,APIError) and (e.status in (401,403,429) or e.reason=='invalid_grant'):delay=300
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1684 |
else:delay=60
|
| 1685 |
self.manager_stop.wait(delay)
|
| 1686 |
finally:
|