v2.45: port dual ingest and probed canvas fixes

README.md

@@ -10,6 +10,36 @@ app_port: 7860
 
 # Reoo88/On — relay-v2.29-smart-mode
 
+## تثبيت الـcanvas على مقاس FLV الحقيقي في v2.45
+
+أظهر القياس الدائم أن وصف مرشح TikTok قد يبقى على مقاس أقدم بينما ترويسة
+الفيديو الفعلية مختلفة: مثلًا `432x864` في الوصف مقابل `640x1280` في
+`STREAM_CONFIG`، و`540x960` مقابل `720x1280`. كان `matchgop` يثبّت الـcanvas
+من الوصف قبل أن يصل المقاس الحقيقي، فيصغر الخرج من أول البث ويبقى ثابتًا.
+
+أصبح فحص المصدر الموجود قبل الإرسال يقرأ أول سجل AVC/HEVC ضمن بادئة محدودة
+(افتراضيًا 256 KiB و4 ثوانٍ، `YOUTUBE_FLV_PROBE_BYTES/SECONDS`). إذا احتوى
+مقاسًا صالحًا، تُحفظ `probed_width/probed_height` بجانب المرشح ويستخدمها
+`lock_gold` بدل الوصف. لا يتغير اسم الجودة أو تثبيت الـrendition. وإذا لم تظهر
+ترويسة مقاس ضمن الحد، يبقى قبول توقيع FLV والسلوك السابق بلا تغيير.
+
+هذا تصحيح لاختيار الـcanvas فقط؛ لا يغيّر `copy` أو `tee/global_header` أو
+القص ومعدل البت، ولا يغيّر مهلة بدء YouTube أو الاسترداد. يغطي
+`test_source_probe.py` التجزئة، والترويسة غير الصالحة أصغر من 32px، وعدم
+إعادة كتابة وصف المرشح، وثبات المسار القديم عند غياب المقاس.
+
+## توحيد ترويسة H.264 مع الاستقبال المزدوج في v2.44
+
+عند إعادة الترميز والنشر المتزامن إلى الاستقبالين الأساسي والاحتياطي عبر FFmpeg
+`tee`، يضيف المرسِل الآن `-flags:v +global_header`. بذلك يقرأ كل مستقبل سجل
+SPS/PPS نفسه منذ البداية بدل أن يلتقطا إعدادَي عرض مختلفين. الدليل الإنتاجي كان
+`resolutionMismatch [error] Bad secondary stream` مع تغيّر حجم الصورة المعروضة،
+وهو نمط يطابق حالة FFmpeg الموثقة ‎#11527 لواجهة `tee` مع يوتيوب.
+
+لا يغيّر الإصلاح المقاس أو فلاتر القص أو معدل البت، ولا يضيف الترويسة إلى مسار
+`copy` الذي يحمل سجل المفكك الأصلي أصلًا. ويبقى النشر المفرد بلا تغيير. يغطي
+`test_ingest_routes.py` وجود الترويسة مع الترميز المزدوج وعدم إضافتها إلى النسخ.
+
 ## تنظيف تلقائي عند غياب البث في v2.27
 
 عند غياب أي بث بشكل كامل ومستمر لمدة `YOUTUBE_IDLE_RESET_SECONDS` (الافتراضي

ingest.py

@@ -6,7 +6,7 @@ import os, threading, time
 from datetime import datetime
 from zoneinfo import ZoneInfo
 
-APP_VERSION = 'relay-v2.43-canvas-lock'
+APP_VERSION = 'relay-v2.45-probed-canvas'
 
 
 def env_rtmp_destination():

relay.py

@@ -469,8 +469,15 @@ class Sender:
             return (self._gop_w, self._gop_h)
         if candidate and not (w and h):
             try:
-                w = int(candidate.get('width') or 0)
-                h = int(candidate.get('height') or 0)
+                # _select_source attaches dimensions parsed from the real FLV decoder record.
+                # Prefer them over TikTok's rendition label, which can remain at an earlier
+                # SD size while the served picture is already HD. Explicit w/h still wins.
+                pw,ph=int(candidate.get('probed_width') or 0),int(candidate.get('probed_height') or 0)
+                if pw>=32 and ph>=32:
+                    w,h=pw,ph
+                else:
+                    w = int(candidate.get('width') or 0)
+                    h = int(candidate.get('height') or 0)
             except (TypeError, ValueError):
                 w = h = 0
         try:
@@ -710,6 +717,12 @@ class Sender:
                 # يعني تسليمًا فعليًا، بلا مخزن يكذب على الحارس.
                 legs=self._dual_legs(destination,extra_destinations)
                 if legs:
+                    # Each tee leg opens its own FLV muxer. With x264, global extradata
+                    # makes both YouTube ingests receive the same SPS/PPS from the start;
+                    # without it YouTube can report resolutionMismatch / Bad secondary
+                    # stream and change the displayed aspect ratio (FFmpeg ticket #11527).
+                    # Copy already carries the bridge's decoder record and is unchanged.
+                    if mode!='copy':video+=['-flags:v','+global_header']
                     output=['-f','tee',self._tee_spec(legs,enhanced)]
                 else:
                     output=['-flvflags','no_duration_filesize','-f','flv']
@@ -1208,7 +1221,9 @@ class Sender:
         tele_meta={'candidate_label':candidate.get('label'),'candidate_brief':candidate.get('brief'),
                    'media':candidate.get('media'),'codec':candidate.get('codec'),
                    'quality_name':candidate.get('quality_name'),'width':candidate.get('width'),
-                   'height':candidate.get('height'),'declared_bitrate_kbps':candidate.get('bitrate')}
+                   'height':candidate.get('height'),'probed_width':candidate.get('probed_width'),
+                   'probed_height':candidate.get('probed_height'),
+                   'declared_bitrate_kbps':candidate.get('bitrate')}
         if isinstance(self.telemetry_context,dict):
             tele_meta.update(self.telemetry_context)
         self.telemetry_key=telemetry_start(user,tele_meta)

test_ingest_routes.py

@@ -229,6 +229,22 @@ class DualIngestTests(unittest.TestCase):
         args=s.command(self.primary,{'codec':'h265'},extra_destinations=[self.backup])
         self.assertEqual(' '.join(args).count('rtmp_enhanced_codecs=hvc1'),2)
 
+    def test_transcoded_tee_uses_global_video_header_on_both_flv_legs(self):
+        # Production evidence: YouTube resolutionMismatch / Bad secondary stream
+        # while one x264 sender fed primary+backup through tee. FFmpeg #11527
+        # documents the matching aspect-ratio symptom and the global-header fix.
+        s=Sender('/bin/true',lambda *a:None);s.mode_override='matchgop'
+        args=s.command(self.primary,{'codec':'h264','width':540,'height':960},
+                       extra_destinations=[self.backup])
+        self.assertIn('tee',args)
+        self.assertEqual(args[args.index('-flags:v')+1],'+global_header')
+
+    def test_copy_tee_does_not_add_encoder_only_global_header_flag(self):
+        s=Sender('/bin/true',lambda *a:None);s.mode_override='copy'
+        args=s.command(self.primary,{'codec':'h264'},extra_destinations=[self.backup])
+        self.assertIn('tee',args)
+        self.assertNotIn('-flags:v',args)
+
     def test_command_single_destination_is_unchanged(self):
         s=Sender('/bin/true',lambda *a:None);s.mode_override='copy'
         args=s.command(self.primary,{'codec':'h264'})

test_smart_mode.py

@@ -72,13 +72,15 @@ class CopyRecoveryTests(unittest.TestCase):
 
 class CanvasCapTests(unittest.TestCase):
     def test_matchgop_canvas_is_capped_at_720_class(self):
-        # 960x1920 must fit inside the 720x1280 box (aspect preserved -> the
-        # canvas is 640x1280), never re-encode at the full 1920 height.
+        # 960x1920 is capped to a 640x1280 canvas, never re-encoded at the
+        # full 1920 height. Since v2.43 this tall 1:2 picture fills that canvas
+        # and center-crops the excess height instead of adding pillarbox bars.
         sender = Sender('/bin/true', lambda *a: None)
         sender.mode_override = 'matchgop'
         args = sender.command('out.flv', {'codec': 'h265', 'width': 960, 'height': 1920})
         vf = args[args.index('-vf') + 1]
-        self.assertIn('pad=640:1280', vf)
+        self.assertIn('crop=640:1280', vf)
+        self.assertNotIn('pad=640:1280', vf)
         self.assertNotIn('1920', vf)
 
     def test_safe_size_mapping_still_applies(self):

test_source_probe.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""Deterministic FLV pre-start metadata tests; no network or FFmpeg required."""
+import os
+import pathlib
+import sys
+import tempfile
+
+ROOT=pathlib.Path(sys.argv[1] if len(sys.argv)>1 else pathlib.Path(__file__).parent).resolve()
+sys.path.insert(0,str(ROOT))
+os.environ.setdefault('PASSWORD','test-pass-not-for-prod')
+os.environ.setdefault('DATA_DIR',tempfile.mkdtemp(prefix='source-probe-test-'))
+os.environ['YOUTUBE_FLV_PROBE_SECONDS']='0.2'
+
+import youtube_manager as manager_module  # noqa: E402
+from relay import Sender  # noqa: E402
+from stream_monitor import video_dimensions  # noqa: E402
+
+passed=failed=0
+def check(name,ok,detail=''):
+    global passed,failed
+    if ok:
+        passed+=1;print('PASS ',name)
+    else:
+        failed+=1;print('FAIL ',name,detail)
+
+
+class Bits:
+    def __init__(self):self.data=[]
+    def bit(self,value):self.data.append(1 if value else 0)
+    def bits(self,value,count):
+        for shift in range(count-1,-1,-1):self.bit((value>>shift)&1)
+    def ue(self,value):
+        code=value+1;size=code.bit_length()
+        self.bits(0,size-1);self.bits(code,size)
+    def rbsp(self):
+        self.bit(1)
+        while len(self.data)%8:self.bit(0)
+        raw=bytes(sum(self.data[i+j]<<(7-j) for j in range(8)) for i in range(0,len(self.data),8))
+        out=bytearray();zeros=0
+        for byte in raw:
+            if zeros>=2 and byte<=3:
+                out.append(3);zeros=0
+            out.append(byte);zeros=zeros+1 if byte==0 else 0
+        return bytes(out)
+
+
+def avcc(width,height):
+    """Minimal baseline H.264 SPS/PPS with uncropped, frame-only 4:2:0 dimensions."""
+    assert width%16==0 and height%16==0
+    b=Bits();b.bits(66,8);b.bits(0,8);b.bits(30,8)  # profile/constraints/level
+    b.ue(0);b.ue(0);b.ue(0);b.ue(0);b.ue(1);b.bit(0)
+    b.ue(width//16-1);b.ue(height//16-1)
+    b.bit(1);b.bit(1);b.bit(0);b.bit(0)
+    sps=b'\x67'+b.rbsp();pps=bytes.fromhex('68ce0fc8')
+    return (bytes([1,66,0,30,0xff,0xe1])+len(sps).to_bytes(2,'big')+sps+
+            b'\x01'+len(pps).to_bytes(2,'big')+pps)
+
+
+def tag(config,ts=0):
+    body=b'\x17\x00\x00\x00\x00'+config;size=len(body)
+    header=(b'\x09'+size.to_bytes(3,'big')+bytes([
+        (ts>>16)&255,(ts>>8)&255,ts&255,(ts>>24)&255,0,0,0]))
+    return header+body+(11+size).to_bytes(4,'big')
+
+
+def flv(*tags):
+    return b'FLV\x01\x05\x00\x00\x00\x09\x00\x00\x00\x00'+b''.join(tags)
+
+
+class Response:
+    def __init__(self,chunks):self.chunks=list(chunks);self.closed=False
+    def raise_for_status(self):return None
+    def iter_content(self):yield from self.chunks
+    def close(self):self.closed=True
+
+
+def manager():
+    obj=manager_module.ManagedYouTube.__new__(manager_module.ManagedYouTube)
+    obj.headers_for=lambda user:{}
+    obj._unstable={};obj.starve={};obj._skip_probe_once={};obj._probe_note=''
+    obj.logs=[];obj.log=lambda user,text:obj.logs.append(str(text))
+    return obj
+
+
+def candidate(width=432,height=864):
+    return {'url':'https://media.example.test/live.flv','media':'flv','codec':'h264',
+            'quality_name':'hd','label':'hd','brief':'hd H264','width':width,'height':height}
+
+
+def probe_with(chunks,cand=None):
+    obj=manager();responses=[]
+    original=manager_module.requests.get
+    def fake_get(*args,**kwargs):
+        response=Response(chunks);responses.append(response);return response
+    manager_module.requests.get=fake_get
+    try:result=obj.probe('alice',cand or candidate())
+    finally:manager_module.requests.get=original
+    return result,responses
+
+
+small=avcc(16,16);real=avcc(640,1280)
+check('test SPS generator carries exact dimensions',
+      video_dimensions('avc1',small)==(16,16) and video_dimensions('avc1',real)==(640,1280))
+
+blob=flv(tag(real));chunks=[blob[i:i+7] for i in range(0,len(blob),7)]
+result,responses=probe_with(chunks)
+check('fragmented FLV probe returns actual decoder dimensions',
+      isinstance(result,dict) and (result.get('width'),result.get('height'))==(640,1280),str(result))
+check('FLV probe closes the source response',bool(responses) and responses[0].closed)
+
+blob=flv(tag(small),tag(real));split=13+len(tag(small))
+result,_=probe_with([blob[:split],blob[split:]])
+check('sub-32px startup config is not accepted as the final canvas',
+      isinstance(result,dict) and (result.get('width'),result.get('height'))==(640,1280),str(result))
+
+result,responses=probe_with([b'FLV'])
+check('signature-only FLV keeps the old truthy fallback',result is True,str(result))
+check('signature-only fallback also closes the response',bool(responses) and responses[0].closed)
+result,_=probe_with([b'NOT AN FLV'])
+check('invalid FLV signature is still rejected',result is False,str(result))
+
+obj=manager();obj.probe=lambda user,c:{'width':640,'height':1280,'codec':'avc1'}
+selected=obj._select_source('alice',{'sources':[candidate()]})
+check('source selection retains a usable candidate',len(selected)==1,str(selected))
+if selected:
+    chosen=selected[0]
+    check('actual dimensions are attached without rewriting the rendition label',
+          (chosen.get('width'),chosen.get('height'))==(432,864) and
+          (chosen.get('probed_width'),chosen.get('probed_height'))==(640,1280) and
+          obj._source_pin(chosen)==obj._source_pin(candidate()),str(chosen))
+    sender=Sender('ffmpeg',lambda *args:None)
+    check('matchgop locks the actual HD canvas instead of the stale SD label',
+          sender.lock_gold(chosen)==(720,1280),str(sender.lock_gold(chosen)))
+    check('a declared/probed mismatch is surfaced in operational logs',
+          any('640x1280' in line and '432x864' in line for line in obj.logs),str(obj.logs))
+
+fallback=Sender('ffmpeg',lambda *args:None)
+check('without decoder metadata the old declared-canvas fallback is unchanged',
+      fallback.lock_gold(candidate())==(432,864),str(fallback.lock_gold(candidate())))
+
+print('source probe tests: %d passed, %d failed'%(passed,failed))
+raise SystemExit(1 if failed else 0)

verify.sh

@@ -30,6 +30,7 @@ python3 test_source_reconnect.py "$ROOT"
 python3 test_ingest_routes.py "$ROOT"
 python3 test_transition_ingest.py "$ROOT"
 python3 test_flv_bridge.py "$ROOT"
+python3 test_source_probe.py "$ROOT"
 python3 test_adaptive_format.py "$ROOT"
 python3 test_completion.py "$ROOT"
 python3 test_telemetry.py "$ROOT"

youtube_manager.py

@@ -8,6 +8,7 @@ from zoneinfo import ZoneInfo
 from urllib.parse import urlparse, urlunparse, quote
 from curl_cffi import requests
 from relay import RelayStartError, Sender, SourceStalled, source_http_headers
+from flv_bridge import FLVBridge
 from stream_monitor import log_event
 from telemetry import record_event as telemetry_event
 import inspect
@@ -1552,11 +1553,16 @@ class ManagedYouTube:
     def _source_magic(media):
         return b'#EXTM3U' if media=='hls' else b'FLV'
     def probe(self,user,candidate):
-        """Check the payload's first bytes before creating any public scheduled event.
+        """Validate the payload and, for FLV, read its real decoder canvas when readily available.
 
-        An FLV rendition must start with the 'FLV' signature, an HLS one with '#EXTM3U'
-        (TikTok's playlists are plain media playlists). Trusting the URL suffix — or the
-        caller's label — is exactly what made every .m3u8 source look broken.
+        An FLV rendition must start with the 'FLV' signature, an HLS one with '#EXTM3U'.
+        TikTok's rendition label can describe a different size from the AVC/HEVC configuration
+        actually served. The existing pre-start HTTP probe already opens the source, so parse its
+        first decoder record here instead of locking ``matchgop`` to that unreliable label.
+
+        The return remains truthy/falsy for existing callers. A mapping with ``width``, ``height``
+        and ``codec`` is returned when a real (>=32px) configuration arrives within the bounded
+        prefix; plain True preserves the old signature-only behavior when it does not.
         """
         media=(candidate or {}).get('media') or 'flv'
         want=self._source_magic(media);got=bytearray()
@@ -1564,10 +1570,36 @@ class ManagedYouTube:
             headers=source_http_headers(candidate.get('url'),user,self.headers_for),stream=True,timeout=20)
         try:
             r.raise_for_status()
+            if media=='hls':
+                for chunk in r.iter_content():
+                    got+=chunk
+                    if len(got)>=len(want):return bytes(got).startswith(b'#EXTM3U')
+                return False
+
+            try:max_bytes=max(4096,int(os.getenv('YOUTUBE_FLV_PROBE_BYTES','262144')))
+            except (TypeError,ValueError):max_bytes=262144
+            try:probe_seconds=max(0.2,float(os.getenv('YOUTUBE_FLV_PROBE_SECONDS','4')))
+            except (TypeError,ValueError):probe_seconds=4.0
+            # Observational parser only: probing must not duplicate STREAM_CONFIG or bitrate
+            # events in the persistent operational log. The real sender has its own bridge.
+            bridge=FLVBridge(user=user,event_logger=lambda *args:None)
+            deadline=time.monotonic()+probe_seconds;seen=0
             for chunk in r.iter_content():
-                got+=chunk
-                if len(got)>=len(want):return bytes(got[:len(want)])==want or (media=='hls' and bytes(got).startswith(b'#EXTM3U'))
-            return False
+                if not chunk:continue
+                seen+=len(chunk)
+                if len(got)<len(want):got+=chunk[:len(want)-len(got)]
+                if len(got)>=len(want) and bytes(got[:len(want)])!=want:return False
+                try:bridge.feed(chunk)
+                except (ValueError,IndexError):
+                    # A valid signature with a truncated prefix retains the old fallback: the
+                    # sender will try the candidate and its watchdog will classify any failure.
+                    return bytes(got[:len(want)])==want
+                size=bridge.video_size
+                if size and min(size)>=32:
+                    return {'width':int(size[0]),'height':int(size[1]),
+                            'codec':str(bridge.video_codec or '')}
+                if seen>=max_bytes or time.monotonic()>=deadline:break
+            return len(got)>=len(want) and bytes(got[:len(want)])==want
         finally:r.close()
     def codec_probe(self,candidate):
         """ffprobe the real payload; TikTok's labels describe the block, not what is served now.
@@ -1742,7 +1774,21 @@ class ManagedYouTube:
         for c in ordered:
             tagged={**c,'user':user}
             try:
-                if self.probe(user,c):
+                probed=self.probe(user,c)
+                if probed:
+                    if isinstance(probed,dict):
+                        try:w,h=int(probed.get('width') or 0),int(probed.get('height') or 0)
+                        except (TypeError,ValueError):w=h=0
+                        if w>=32 and h>=32:
+                            # Keep the rendition label/pin unchanged; these fields describe the
+                            # actual decoder record and are used only for the constant canvas.
+                            tagged['probed_width'],tagged['probed_height']=w,h
+                            tagged['probed_codec']=str(probed.get('codec') or '')
+                            try:declared=(int(c.get('width') or 0),int(c.get('height') or 0))
+                            except (TypeError,ValueError):declared=(0,0)
+                            if declared!=(w,h):
+                                self.log(user,'[SOURCE] FLV config %sx%s يخالف وصف المرشح %sx%s؛ سيُثبَّت الخرج على المقاس الحقيقي'%(
+                                    w,h,declared[0],declared[1]))
                     usable.append(tagged)
             except Exception as e:
                 self._probe_note=self.safe_error(e)