File size: 2,249 Bytes
fac8bb8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | """Read-only source audit. Never modifies the supplied source directory."""
import argparse
from collections import Counter
from pathlib import Path
import pyarrow.parquet as pq
from common import dump, sha, now, MAX_SOURCE_BYTES, MAX_FILE_BYTES
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--source', default='D:/AI_DATA/wiki_clean/thinking/english')
ap.add_argument('--out', default='artifacts/source_manifest.json')
args = ap.parse_args()
files, excluded, total = [], [], 0
for p in sorted(Path(args.source).glob('*.parquet')):
size = p.stat().st_size
if size >= MAX_FILE_BYTES or total + size >= MAX_SOURCE_BYTES:
excluded.append({'name': p.name, 'bytes': size, 'reason': 'strict byte limit'})
continue
pf = pq.ParquetFile(p)
assert {'source','kind','prompt','think','answer'} <= set(pf.schema_arrow.names)
row = next(pf.iter_batches(batch_size=1)).to_pylist()[0]
owner, repo = p.name.split('__')[:2]
files.append({'name': p.name, 'bytes': size, 'sha256': sha(p),
'rows': pf.metadata.num_rows, 'row_groups': pf.num_row_groups,
'source': row['source'], 'upstream_repo': owner + '/' + repo,
'source_revision': None, 'local_mtime_ns': p.stat().st_mtime_ns})
total += size
print('AUDIT', len(files), p.name, size, flush=True)
manifest = {'created_utc': now(), 'source_directory': str(Path(args.source).resolve()),
'source_revision_note': 'Historical upstream revisions were not retained in these cleaned local files. Local SHA-256 values define this exact snapshot; current upstream revisions are not claimed as historical identities.',
'limits': {'total_bytes_strictly_less_than': MAX_SOURCE_BYTES,
'single_file_bytes_strictly_less_than': MAX_FILE_BYTES},
'file_count': len(files), 'total_bytes': total,
'total_rows': sum(x['rows'] for x in files),
'files': files, 'excluded': excluded}
dump(args.out, manifest)
print('AUDIT_COMPLETE', len(files), total, manifest['total_rows'], flush=True)
if __name__ == '__main__': main()
|