| """Collect existing source-scene videos as verified physical copies; never move originals.""" |
| from __future__ import annotations |
| from concurrent.futures import ThreadPoolExecutor, as_completed |
| from datetime import datetime, timezone |
| import argparse |
| import csv |
| import hashlib |
| import html |
| import json |
| import os |
| from pathlib import Path |
| import re |
| import shutil |
| import threading |
| import time |
| import urllib.parse |
|
|
| HERE = Path(__file__).resolve().parent |
| REPO = HERE.parents[1] |
| DEST = Path('Z:/ryli/UE_ReferenceVideos_All_20260925') |
| LOCK = threading.Lock() |
| PROGRESS = {} |
|
|
|
|
| def utc(): |
| return datetime.now(timezone.utc).isoformat() |
|
|
|
|
| def read(p): |
| return json.loads(Path(p).read_text('utf-8-sig')) |
|
|
|
|
| def save(p, data): |
| p = Path(p) |
| p.parent.mkdir(parents=True, exist_ok=True) |
| tmp = p.with_name(p.name + '.writing') |
| tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') |
| os.replace(tmp, p) |
|
|
|
|
| def digest(p): |
| h = hashlib.sha256() |
| with Path(p).open('rb') as f: |
| for chunk in iter(lambda: f.read(8 * 1024 * 1024), b''): |
| h.update(chunk) |
| return h.hexdigest() |
|
|
|
|
| def safe_title(s): |
| return re.sub(r'[<>:"/\\|?*\x00-\x1f]', '_', s).strip(' .') |
|
|
|
|
| def plan(): |
| records, groups, seen = [], {}, {} |
| sources = [('20260921', Path('Z:/ryli/UE_RecordedRouteVideos_20260921/batch_manifest.json')), |
| ('20260924', Path('Z:/ryli/UE_RecordedRouteVideos_20260924/batch_manifest_v4.json'))] |
| for batch, manifest in sources: |
| for scene in read(manifest)['scenes']: |
| folder = Path(scene['output_dir']) |
| receipt_path = folder / 'video_receipt.json' |
| rec = read(receipt_path) |
| assert rec['status'] == 'complete' and rec.get('partial_diagnostic') is False |
| video = folder / 'route_native.mp4' |
| stat = video.stat() |
| assert stat.st_size == rec['video_bytes'] |
| assert rec['trajectory_sha256'] == scene['trajectory_sha256'] |
| identity = rec['video_sha256'] |
| if identity in seen: |
| seen[identity]['also_listed_in'].append(str(manifest)) |
| continue |
| group_id = scene['scene_id'] |
| if group_id not in groups: |
| groups[group_id] = {'number': len(groups)+1, 'title': scene['title'], 'map_package': scene['map_package']} |
| group = groups[group_id] |
| capture_date = rec['completed_utc'][:10].replace('-', '') |
| relative = f"Scenes/{group['number']:02}_{safe_title(group['title'])}/reference_{capture_date}.mp4" |
| assert relative not in {r['relative_path'] for r in records} |
| row = {'kind': 'continuous_scene_reference', 'scene_id': group_id, 'scene_number': group['number'], |
| 'title': scene['title'], 'batch': batch, 'capture_date': capture_date, |
| 'source_video': str(video), 'relative_path': relative, 'destination_video': str(DEST / relative), |
| 'source_receipt': str(receipt_path), 'source_manifest': str(manifest), 'also_listed_in': [], |
| 'expected_sha256': identity, 'bytes': stat.st_size, 'source_mtime_ns': stat.st_mtime_ns, |
| 'duration_seconds': rec['encoded_duration_seconds'], 'frames': rec['frame_count'], |
| 'fps': rec['fps'], 'width': rec['width'], 'height': rec['height'], |
| 'trajectory_sha256': rec['trajectory_sha256'], 'map_package': rec['map_package'], |
| 'note_zh': '完整原路线，保留停留、快速移动、遮挡和空域；未宣称全部区域或逐帧参考质量通过。', |
| 'source_quality_certified': False} |
| records.append(row) |
| seen[identity] = row |
| archive = Path('Z:/ryli/UE_RecordedRouteVideos_20260920/Overview') |
| rec = read(archive / 'video_receipt.json') |
| p = archive / 'route_native.mp4' |
| stat = p.stat() |
| assert stat.st_size == rec['video_bytes'] and rec['status'] == 'complete' |
| relative = 'Historical/Asset_Display/餐厅资产陈列_20260920.mp4' |
| records.append({'kind': 'historical_asset_display', 'scene_id': rec['scene_id'], 'title': '餐厅资产陈列（历史）', |
| 'source_video': str(p), 'relative_path': relative, 'destination_video': str(DEST / relative), |
| 'source_receipt': str(archive / 'video_receipt.json'), 'source_manifest': None, 'also_listed_in': [], |
| 'expected_sha256': rec['video_sha256'], 'bytes': stat.st_size, 'source_mtime_ns': stat.st_mtime_ns, |
| 'duration_seconds': rec['encoded_duration_seconds'], 'frames': rec['frame_count'], |
| 'fps': rec['fps'], 'width': rec['width'], 'height': rec['height'], |
| 'note_zh': '单体资产陈列地图，不计入19个完整游戏场景。', 'source_quality_certified': False}) |
| cathedral = Path('Z:/ryli/UE_InteractiveReferences_20260917/scenes/City__04_OldCathedral__L_Showcase') |
| for sub, name in [('captures/20260917T093116_569589_ab419328', '教堂_稀疏采样_20260917'), ('reference_01', '教堂_旧地图稀疏采样_reference01')]: |
| folder = cathedral / sub |
| p = folder / 'walkthrough.mp4' |
| timing = read(folder / 'video_timing.json') |
| assert timing['hold_between_recorded_frames'] is True |
| stat = p.stat() |
| relative = f'Historical/Sparse_Frame_Videos/{name}.mp4' |
| records.append({'kind': 'historical_sparse_frame_video', 'scene_id': 'City__04_OldCathedral__L_Showcase', |
| 'title': name, 'source_video': str(p), 'relative_path': relative, 'destination_video': str(DEST / relative), |
| 'source_receipt': str(folder / 'video_timing.json'), 'source_manifest': None, 'also_listed_in': [], |
| 'expected_sha256': None, 'bytes': stat.st_size, 'source_mtime_ns': stat.st_mtime_ns, |
| 'duration_seconds': None, 'source_route_duration_seconds': timing['source_duration_seconds'], |
| 'frames': None, 'native_unique_frames': timing['native_unique_frame_count'], |
| 'fps': timing['output_fps'], 'width': None, 'height': None, |
| 'note_zh': '1596张原生图以保持画面的方式编码，不能当作连续逐帧原生录像。reference01的源地图版本与当前网络地图不同。', |
| 'source_quality_certified': False}) |
| assert len(groups) == 19 and sum(r['kind'] == 'continuous_scene_reference' for r in records) == 22 |
| return {'schema': 'ue-reference-video-consolidation/1', 'planned_utc': utc(), 'destination': str(DEST), |
| 'status': 'planned', 'scene_count': len(groups), 'continuous_scene_video_count': 22, |
| 'historical_video_count': 3, 'total_video_count': len(records), |
| 'scene_video_bytes': sum(r['bytes'] for r in records if r['kind'] == 'continuous_scene_reference'), |
| 'total_bytes': sum(r['bytes'] for r in records), |
| 'scene_duration_seconds': sum(r['duration_seconds'] for r in records if r['kind'] == 'continuous_scene_reference'), |
| 'rows': records, 'source_batches': [str(p) for _, p in sources], |
| 'excluded': [ |
| {'path': 'Z:/ryli/dense_references_20260916', 'status': 'discard', 'reason_zh': '用户已明确弃用整批，不重新纳入或解包；原归档保留。'}, |
| {'path': 'Z:/ryli/GeneratedGames_MatchedRoutes_20260924', 'reason_zh': '生成游戏对照录像，不是原场景参考；其中reference副本与原片重复。'}, |
| {'reason_zh': '诊断短预览、失败尝试、录制缓存副本及生成游戏展示不重复收录。'}], |
| 'copy_policy': 'physical_byte_identical_copies_originals_unchanged', |
| 'quality_scope_zh': '整理和完整性校验；不升级历史质量结论，不剪辑或重新编码。'} |
|
|
|
|
| def progress(row, phase, n): |
| with LOCK: |
| PROGRESS[row['relative_path']] = {'phase': phase, 'bytes': n, 'total': row['bytes'], 'updated_utc': utc()} |
| data = {'status': 'copying_and_verifying', 'updated_utc': utc(), 'pid': os.getpid(), 'files': dict(PROGRESS)} |
| save(HERE / 'PROGRESS.json', data) |
|
|
|
|
| def copy_one(row): |
| start = time.monotonic() |
| source = Path(row['source_video']) |
| target = DEST / row['relative_path'] |
| assert target.resolve().is_relative_to(DEST.resolve()) |
| target.parent.mkdir(parents=True, exist_ok=True) |
| assert source.stat().st_size == row['bytes'] and source.stat().st_mtime_ns == row['source_mtime_ns'] |
| record_file = HERE / 'verified' / (hashlib.sha256(row['relative_path'].encode()).hexdigest()[:16] + '.json') |
| if target.exists(): |
| previous = read(record_file) |
| assert previous['source_video'] == str(source) |
| assert target.stat().st_size == row['bytes'] |
| actual = digest(target) |
| assert actual == previous['sha256'] |
| if row['expected_sha256']: |
| assert actual == row['expected_sha256'] |
| progress(row, 'verified_existing_copy', row['bytes']) |
| return previous |
| temp = target.with_name(target.name + f'.copying_{os.getpid()}') |
| assert not temp.exists(), f'Preserve incomplete attempt: {temp}' |
| h = hashlib.sha256() |
| copied = 0 |
| last = time.monotonic() |
| progress(row, 'copying', 0) |
| with source.open('rb') as inp, temp.open('xb') as out: |
| for chunk in iter(lambda: inp.read(8 * 1024 * 1024), b''): |
| out.write(chunk) |
| h.update(chunk) |
| copied += len(chunk) |
| if time.monotonic() - last > 4: |
| progress(row, 'copying', copied) |
| last = time.monotonic() |
| out.flush() |
| os.fsync(out.fileno()) |
| source_hash = h.hexdigest() |
| assert copied == row['bytes'] |
| assert source.stat().st_size == row['bytes'] and source.stat().st_mtime_ns == row['source_mtime_ns'] |
| if row['expected_sha256']: |
| assert source_hash == row['expected_sha256'], f'Original hash changed: {source}' |
| progress(row, 'verifying_destination_bytes', copied) |
| target_hash = digest(temp) |
| assert target_hash == source_hash |
| assert not target.exists() |
| |
| os.rename(temp, target) |
| result = {'source_video': str(source), 'destination_video': str(target), 'relative_path': row['relative_path'], |
| 'bytes': copied, 'sha256': source_hash, 'source_receipt_hash_matched': bool(row['expected_sha256']), |
| 'source_and_destination_full_bytes_verified': True, 'source_unchanged_during_copy': True, |
| 'verified_utc': utc(), 'elapsed_seconds': round(time.monotonic()-start, 2)} |
| save(record_file, result) |
| progress(row, 'verified', copied) |
| print(json.dumps({'copied': row['title'], 'bytes': copied, 'seconds': result['elapsed_seconds']}, ensure_ascii=False), flush=True) |
| return result |
|
|
|
|
| def publish(data, receipts): |
| result_by_path = {r['relative_path']: r for r in receipts} |
| for row in data['rows']: |
| row['copy_verification'] = result_by_path[row['relative_path']] |
| assert Path(row['destination_video']).stat().st_size == row['bytes'] |
| data.update(status='complete', completed_utc=utc(), source_and_destination_all_full_hashes_verified=True, |
| new_full_media_decode_performed=False, note_zh='完整原生视频沿用原批次解码及审阅；本次重新逐字节核对源与副本。') |
| save(DEST / 'CATALOG.json', data) |
| save(HERE / 'FINAL_CATALOG.json', data) |
| with (DEST / 'CATALOG.csv').open('w', encoding='utf-8-sig', newline='') as f: |
| fields = ['kind', 'title', 'relative_path', 'destination_video', 'source_video', 'duration_seconds', 'frames', 'fps', 'width', 'height', 'bytes', 'sha256', 'note_zh'] |
| writer = csv.DictWriter(f, fieldnames=fields) |
| writer.writeheader() |
| for row in data['rows']: |
| writer.writerow({k: (row['copy_verification']['sha256'] if k == 'sha256' else row.get(k)) for k in fields}) |
| meta = DEST / '_Metadata' |
| meta.mkdir(exist_ok=True) |
| for n, row in enumerate(data['rows'], 1): |
| shutil.copy2(row['source_receipt'], meta / f'{n:02}_original_{Path(row["source_receipt"]).name}') |
| for n, path in enumerate(data['source_batches'], 1): |
| shutil.copy2(path, meta / f'batch_{n}_manifest.json') |
| shutil.copy2(Path('Z:/ryli/dense_references_20260916/DATASET_STATUS.json'), meta / 'DISCARD_DATASET_STATUS.json') |
| save(meta / 'COPY_VERIFICATION.json', receipts) |
| docs = [Path('Z:/ryli/UE_RecordedRouteVideos_20260920/DELIVERY_zh-CN.md'), |
| Path('Z:/ryli/UE_RecordedRouteVideos_20260921/METHOD_zh-CN.md'), |
| Path('Z:/ryli/UE_RecordedRouteVideos_20260924/DELIVERY_zh-CN.md')] |
| for p in docs: |
| shutil.copy2(p, meta / (p.parent.name + '_' + p.name)) |
| scene_rows = [r for r in data['rows'] if r['kind'] == 'continuous_scene_reference'] |
| mins = data['scene_duration_seconds'] / 60 |
| lines = ['# Unreal 原场景参考视频汇总', '', f"共19个真实场景、22条不同完整路线，约{mins:.2f}分钟。另有3条历史视频，分别为1条资产陈列和2条稀疏原生图保持影片。", '', |
| '所有视频均为实际文件副本；已逐字节计算源文件与目标文件摘要，确认一致。没有剪辑、改画面或重新编码。原目录保持不变。', '', |
| '- `Scenes`：正式连续原场景路线视频，每景一个目录；仓库、办公室和学校保留两次不同采样。', |
| '- `Historical`：资产陈列和旧教堂稀疏采样，不计入22条连续场景影片。', |
| '- `index.html`：场景清单、搜索和直接播放。', |
| '- `CATALOG.csv` / `CATALOG.json`：绝对路径、来源、时长、大小及文件摘要。', |
| '- `_Metadata`：原回执、批次清单、复制验证与历史使用限制。', '', |
| '完整录像包含原路线的遮挡、空域和快速移动，不代表全地图覆盖或逐帧质量合格。学校9月24日采用已完成原生帧恢复编码，原引擎退出码未知的限制沿用原回执。', '', |
| '9月16日整批discard结果仍在 `Z:\\ryli\\dense_references_20260916`，按此前决定保持弃用；未解包、未重新混入参考输入。诊断短片、生成游戏视频与重复副本不计入本目录场景参考。', '', |
| '| 场景 | 录像日期 | 秒数 | 相对路径 |', '| --- | --- | ---: | --- |'] |
| for r in scene_rows: |
| lines.append(f"| {r['title']} | {r['capture_date']} | {r['duration_seconds']:.3f} | [{r['relative_path']}]({urllib.parse.quote(r['relative_path'])}) |") |
| (DEST / 'README_zh-CN.md').write_text('\n'.join(lines)+'\n', encoding='utf-8') |
| table = [] |
| for r in data['rows']: |
| kind = '连续场景' if r['kind'] == 'continuous_scene_reference' else '历史资料' |
| duration = f"{r['duration_seconds']:.2f} 秒" if r.get('duration_seconds') is not None else '约398.49秒原路线；稀疏帧保持' |
| path = urllib.parse.quote(r['relative_path'], safe='/') |
| table.append(f'<tr data-kind="{kind}"><td>{html.escape(r["title"])}</td><td>{kind}</td><td>{r.get("capture_date", "历史")}</td><td>{duration}</td><td><a href="{path}" target="_blank">打开视频</a> <button data-src="{path}" data-title="{html.escape(r["title"],quote=True)}">页内播放</button><small>{html.escape(r["relative_path"])}</small></td></tr>') |
| page = '''<!doctype html><html lang="zh-CN"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Unreal 原场景参考视频汇总</title><style> |
| body{font:16px/1.8 "Microsoft YaHei",sans-serif;margin:0;background:#eef2f6;color:#203249}main{max-width:1200px;margin:32px auto;padding:28px;background:white;border-radius:10px}h1{margin-top:0}.summary{background:#e7f1f7;padding:14px 20px;border-left:4px solid #397493}a{color:#176290}input,select,button{font:inherit;padding:7px;margin:8px 5px 8px 0}table{width:100%;border-collapse:collapse;font-size:14px}th,td{border-bottom:1px solid #ccd7e1;padding:10px;text-align:left}th{background:#eaf0f5}small{display:block;font-size:11px;overflow-wrap:anywhere;color:#617185}video{width:100%;max-height:70vh;background:#111}#viewer{margin:24px 0}#viewer[hidden]{display:none}footer{font-size:13px;color:#536577;margin-top:25px}@media(max-width:750px){main{margin:0;padding:14px}table{font-size:11px}th,td{padding:5px}h1{font-size:24px}} |
| </style><main><h1>Unreal 原场景参考视频汇总</h1>''' |
| page += f'<p class="summary"><b>19 个真实场景 · 22 条完整路线 · {mins:.2f} 分钟</b><br>仓库、办公室、学校各保留两条采样。另列3条历史资料。全部为字节一致的实际视频副本，原文件保留。</p>' |
| page += '<p>这里只汇总原场景采样；未混入生成游戏演示。录像完整性与画面可用性分别判断，遮挡和空域仍需按用途筛选。</p><p><a href="README_zh-CN.md">使用说明</a> · <a href="CATALOG.csv">下载清单</a> · <a href="CATALOG.json">来源与校验记录</a></p><input id="search" placeholder="搜索场景或日期"><select id="kind"><option>全部</option><option>连续场景</option><option>历史资料</option></select><p id="count"></p><section id="viewer" hidden><h2 id="now"></h2><video id="player" controls preload="none"></video></section><table><thead><tr><th>场景</th><th>类别</th><th>日期</th><th>时长</th><th>视频文件</th></tr></thead><tbody>' |
| page += ''.join(table) |
| page += '</tbody></table><footer>9月16日整批弃用采样未重新纳入，位置：Z:\\ryli\\dense_references_20260916。历史资产陈列、稀疏图像影片不计入连续场景数量。复制校验不代表已经通过全部参考质量验收。</footer></main><script>function filter(){const q=document.getElementById("search").value.toLowerCase(),k=document.getElementById("kind").value;let n=0;document.querySelectorAll("tbody tr").forEach(r=>{r.hidden=!r.textContent.toLowerCase().includes(q)||(k!=="全部"&&r.dataset.kind!==k);if(!r.hidden)n++;});document.getElementById("count").textContent="当前显示 "+n+" 条视频";}document.getElementById("search").oninput=filter;document.getElementById("kind").onchange=filter;document.querySelectorAll("button[data-src]").forEach(b=>b.onclick=()=>{const v=document.getElementById("player");v.pause();v.src=b.dataset.src;document.getElementById("now").textContent=b.dataset.title;document.getElementById("viewer").hidden=false;v.load();document.getElementById("viewer").scrollIntoView({behavior:"smooth"});});filter();</script></html>' |
| (DEST / 'index.html').write_text(page, encoding='utf-8') |
| local = REPO / 'UNREAL_REFERENCE_VIDEOS_zh-CN.md' |
| local.write_text('# Unreal 原场景参考视频汇总\n\n'+f'全部汇总目录：`{DEST}`\n\n共19个真实场景、22条完整参考路线，约{mins:.2f}分钟。另有3条历史资料，单独分类。\n\n[打开汇总网页](Z:/ryli/UE_ReferenceVideos_All_20260925/index.html)\n\n`Scenes` 中每个场景一个文件夹，均为实际视频副本；同一场景不同路线全部保留。原视频未移动或修改。源与目标完整内容摘要已核对。\n\n原9月16日discard数据保持弃用，不混入。完整清单、来源、时长和校验值见汇总目录的CATALOG.csv与CATALOG.json。\n', encoding='utf-8') |
| save(HERE / 'PROGRESS.json', {'status': 'complete', 'completed_utc': utc(), 'pid': os.getpid(), 'files': PROGRESS}) |
| print(json.dumps({'status': 'complete', 'destination': str(DEST), 'scene_count': 19, 'scene_videos': 22, 'historical_videos': 3, 'bytes': data['total_bytes'], 'minutes': mins}, ensure_ascii=False), flush=True) |
|
|
|
|
| if __name__ == '__main__': |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument('--execute', action='store_true') |
| args = parser.parse_args() |
| HERE.mkdir(parents=True, exist_ok=True) |
| if not args.execute: |
| data = plan() |
| save(HERE / 'PLAN.json', data) |
| print(json.dumps({k: v for k, v in data.items() if k not in ['rows', 'excluded']}, ensure_ascii=False)) |
| else: |
| data = read(HERE / 'PLAN.json') |
| assert data['destination'] == str(DEST) |
| assert shutil.disk_usage(DEST.parent).free > data['total_bytes'] + 1024**3 |
| if DEST.exists(): |
| previous = read(DEST / '_Metadata/CONSOLIDATION_PLAN.json') |
| assert previous == data, 'Existing output belongs to a different inventory' |
| else: |
| DEST.mkdir() |
| save(DEST / '_Metadata/CONSOLIDATION_PLAN.json', data) |
| shutil.copy2(Path(__file__), DEST / '_Metadata/executed_collect_reference_videos.py') |
| receipts = [] |
| with ThreadPoolExecutor(max_workers=2) as pool: |
| futures = [pool.submit(copy_one, row) for row in data['rows']] |
| for result in as_completed(futures): |
| receipts.append(result.result()) |
| publish(data, receipts) |
|
|
