| """Read-only source audit. Never modifies the supplied source directory.""" |
| import argparse |
| from collections import Counter |
| from pathlib import Path |
| import pyarrow.parquet as pq |
| from common import dump, sha, now, MAX_SOURCE_BYTES, MAX_FILE_BYTES |
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument('--source', default='D:/AI_DATA/wiki_clean/thinking/english') |
| ap.add_argument('--out', default='artifacts/source_manifest.json') |
| args = ap.parse_args() |
| files, excluded, total = [], [], 0 |
| for p in sorted(Path(args.source).glob('*.parquet')): |
| size = p.stat().st_size |
| if size >= MAX_FILE_BYTES or total + size >= MAX_SOURCE_BYTES: |
| excluded.append({'name': p.name, 'bytes': size, 'reason': 'strict byte limit'}) |
| continue |
| pf = pq.ParquetFile(p) |
| assert {'source','kind','prompt','think','answer'} <= set(pf.schema_arrow.names) |
| row = next(pf.iter_batches(batch_size=1)).to_pylist()[0] |
| owner, repo = p.name.split('__')[:2] |
| files.append({'name': p.name, 'bytes': size, 'sha256': sha(p), |
| 'rows': pf.metadata.num_rows, 'row_groups': pf.num_row_groups, |
| 'source': row['source'], 'upstream_repo': owner + '/' + repo, |
| 'source_revision': None, 'local_mtime_ns': p.stat().st_mtime_ns}) |
| total += size |
| print('AUDIT', len(files), p.name, size, flush=True) |
| manifest = {'created_utc': now(), 'source_directory': str(Path(args.source).resolve()), |
| 'source_revision_note': 'Historical upstream revisions were not retained in these cleaned local files. Local SHA-256 values define this exact snapshot; current upstream revisions are not claimed as historical identities.', |
| 'limits': {'total_bytes_strictly_less_than': MAX_SOURCE_BYTES, |
| 'single_file_bytes_strictly_less_than': MAX_FILE_BYTES}, |
| 'file_count': len(files), 'total_bytes': total, |
| 'total_rows': sum(x['rows'] for x in files), |
| 'files': files, 'excluded': excluded} |
| dump(args.out, manifest) |
| print('AUDIT_COMPLETE', len(files), total, manifest['total_rows'], flush=True) |
|
|
| if __name__ == '__main__': main() |
|
|
|
|
